diff options
| author | Arne H Juul <arnej@yahoo-inc.com> | 2017-01-06 09:41:42 +0100 |
|---|---|---|
| committer | Arne H Juul <arnej@yahoo-inc.com> | 2017-01-06 09:41:42 +0100 |
| commit | dd8e1ccb0efb7d0a513d2349bdd778b9dcd7a4c9 (patch) | |
| tree | 143ab3d76265b8e1baf2e56c9deef79971a12f5c /documentapi | |
| parent | 56faf25a6f550d82b39fb1e1c4b838c630e5c8bf (diff) | |
fix whitespace only
Diffstat (limited to 'documentapi')
48 files changed, 8643 insertions, 8643 deletions
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java index 183e4ea63d3..f759068cce2 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java @@ -10,7 +10,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; * <p>A session for asynchronous access to a document repository. * This class provides document repository writes and random access with high * throughput.</p> - * + * * <p>All operations which are <i>accepted</i> by an async session will cause one or more * {@link Response responses} to be returned within the timeout limit. When an operation fails, * the response will contain the argument which was submitted to the operation.</p> diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java index 701fafbab06..b5caa2f3812 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java @@ -1,38 +1,38 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi;
-
-import com.yahoo.document.config.DocumentmanagerConfig;
-
-import java.util.Optional;
-
-/**
- * Superclass of the classes which contains the parameters for creating or opening a document access.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class DocumentAccessParams {
-
- /** The id to resolve to document manager config. Not needed if the config is passed here */
- private String documentManagerConfigId = "client";
-
- /** The document manager config, or empty if not provided (in which case a subscription must be created) */
- private Optional<DocumentmanagerConfig> documentmanagerConfig = Optional.empty();
-
- /** Returns the config id that the document manager should subscribe to. */
- public String getDocumentManagerConfigId() { return documentManagerConfigId; }
-
- /** Returns the document manager config to use, or empty if it it necessary to subscribe to get it */
- public Optional<DocumentmanagerConfig> documentmanagerConfig() { return documentmanagerConfig; }
-
- /** Sets the config id that the document manager should subscribe to. */
- public DocumentAccessParams setDocumentManagerConfigId(String configId) {
- documentManagerConfigId = configId;
- return this;
- }
-
- public DocumentAccessParams setDocumentmanagerConfig(DocumentmanagerConfig documentmanagerConfig) {
- this.documentmanagerConfig = Optional.of(documentmanagerConfig);
- return this;
- }
-
-}
\ No newline at end of file +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.config.DocumentmanagerConfig; + +import java.util.Optional; + +/** + * Superclass of the classes which contains the parameters for creating or opening a document access. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class DocumentAccessParams { + + /** The id to resolve to document manager config. Not needed if the config is passed here */ + private String documentManagerConfigId = "client"; + + /** The document manager config, or empty if not provided (in which case a subscription must be created) */ + private Optional<DocumentmanagerConfig> documentmanagerConfig = Optional.empty(); + + /** Returns the config id that the document manager should subscribe to. */ + public String getDocumentManagerConfigId() { return documentManagerConfigId; } + + /** Returns the document manager config to use, or empty if it it necessary to subscribe to get it */ + public Optional<DocumentmanagerConfig> documentmanagerConfig() { return documentmanagerConfig; } + + /** Sets the config id that the document manager should subscribe to. */ + public DocumentAccessParams setDocumentManagerConfigId(String configId) { + documentManagerConfigId = configId; + return this; + } + + public DocumentAccessParams setDocumentmanagerConfig(DocumentmanagerConfig documentmanagerConfig) { + this.documentmanagerConfig = Optional.of(documentmanagerConfig); + return this; + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java index 24b68613208..2db5542de23 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi;
-
-/**
- * Parameters for creating a synchronous session
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class SyncParameters extends Parameters {
- // empty
-}
+package com.yahoo.documentapi; + +/** + * Parameters for creating a synchronous session + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SyncParameters extends Parameters { + // empty +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java index f864898fb5b..252ac35b8cd 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java @@ -1,101 +1,101 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi;
-
-import com.yahoo.document.Document;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.DocumentPut;
-import com.yahoo.document.DocumentRemove;
-import com.yahoo.document.DocumentType;
-import com.yahoo.document.DocumentUpdate;
-import com.yahoo.document.TestAndSetCondition;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-
-/**
- * <p>A session for synchronous access to a document repository. This class
- * provides simple document access where throughput is not a concern.</p>
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public interface SyncSession extends Session {
-
- /**
- * <p>Puts a document. When this method returns, the document is safely
- * received. This enables setting condition compared to using Document.</p>
- *
- * @param documentPut The DocumentPut operation
- */
- void put(DocumentPut documentPut);
-
- /**
- * <p>Puts a document. When this method returns, the document is safely
- * received.</p>
- *
- * @param documentPut The DocumentPut operation
- * @param priority The priority with which to perform this operation.
- */
- void put(DocumentPut documentPut, DocumentProtocol.Priority priority);
-
- /**
- * <p>Gets a document.</p>
- *
- * @param id The id of the document to get.
- * @return The known document having this id, or null if there is no
- * document having this id.
- * @throws UnsupportedOperationException Thrown if this access does not
- * support retrieving.
- */
- Document get(DocumentId id);
-
- /**
- * <p>Gets a document.</p>
- *
- * @param id The id of the document to get.
- * @param fieldSet A comma-separated list of fields to retrieve
- * @param priority The priority with which to perform this operation.
- * @return The known document having this id, or null if there is no
- * document having this id.
- * @throws UnsupportedOperationException Thrown if this access does not
- * support retrieving.
- */
- Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority);
-
- /**
- * <p>Removes a document if it is present and condition is fulfilled.</p>
- * @param documentRemove document to delete
- * @return true If the document with this id was removed, false otherwise.
- */
- boolean remove(DocumentRemove documentRemove);
-
- /**
- * <p>Removes a document if it is present.</p>
- *
- * @param documentRemove Document remove operation
- * @param priority The priority with which to perform this operation.
- * @return true If the document with this id was removed, false otherwise.
- * @throws UnsupportedOperationException Thrown if this access does not
- * support removal.
- */
- boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority);
-
- /**
- * <p>Updates a document.</p>
- *
- * @param update The updates to perform.
- * @return True, if the document was found and updated.
- * @throws UnsupportedOperationException Thrown if this access does not
- * support update.
- */
- boolean update(DocumentUpdate update);
-
- /**
- * <p>Updates a document.</p>
- *
- * @param update The updates to perform.
- * @param priority The priority with which to perform this operation.
- * @return True, if the document was found and updated.
- * @throws UnsupportedOperationException Thrown if this access does not
- * support update.
- */
- boolean update(DocumentUpdate update, DocumentProtocol.Priority priority);
-
-}
+package com.yahoo.documentapi; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; + +/** + * <p>A session for synchronous access to a document repository. This class + * provides simple document access where throughput is not a concern.</p> + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface SyncSession extends Session { + + /** + * <p>Puts a document. When this method returns, the document is safely + * received. This enables setting condition compared to using Document.</p> + * + * @param documentPut The DocumentPut operation + */ + void put(DocumentPut documentPut); + + /** + * <p>Puts a document. When this method returns, the document is safely + * received.</p> + * + * @param documentPut The DocumentPut operation + * @param priority The priority with which to perform this operation. + */ + void put(DocumentPut documentPut, DocumentProtocol.Priority priority); + + /** + * <p>Gets a document.</p> + * + * @param id The id of the document to get. + * @return The known document having this id, or null if there is no + * document having this id. + * @throws UnsupportedOperationException Thrown if this access does not + * support retrieving. + */ + Document get(DocumentId id); + + /** + * <p>Gets a document.</p> + * + * @param id The id of the document to get. + * @param fieldSet A comma-separated list of fields to retrieve + * @param priority The priority with which to perform this operation. + * @return The known document having this id, or null if there is no + * document having this id. + * @throws UnsupportedOperationException Thrown if this access does not + * support retrieving. + */ + Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority); + + /** + * <p>Removes a document if it is present and condition is fulfilled.</p> + * @param documentRemove document to delete + * @return true If the document with this id was removed, false otherwise. + */ + boolean remove(DocumentRemove documentRemove); + + /** + * <p>Removes a document if it is present.</p> + * + * @param documentRemove Document remove operation + * @param priority The priority with which to perform this operation. + * @return true If the document with this id was removed, false otherwise. + * @throws UnsupportedOperationException Thrown if this access does not + * support removal. + */ + boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority); + + /** + * <p>Updates a document.</p> + * + * @param update The updates to perform. + * @return True, if the document was found and updated. + * @throws UnsupportedOperationException Thrown if this access does not + * support update. + */ + boolean update(DocumentUpdate update); + + /** + * <p>Updates a document.</p> + * + * @param update The updates to perform. + * @param priority The priority with which to perform this operation. + * @return True, if the document was found and updated. + * @throws UnsupportedOperationException Thrown if this access does not + * support update. + */ + boolean update(DocumentUpdate update, DocumentProtocol.Priority priority); + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java index cde434df141..30bed329918 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java @@ -1,797 +1,797 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi;
-
-import com.yahoo.document.BucketId;
-import com.yahoo.document.BucketIdFactory;
-import com.yahoo.document.select.BucketSelector;
-import com.yahoo.document.select.parser.ParseException;
-import com.yahoo.log.LogLevel;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.logging.Logger;
-
-/**
- * <p>Enables transparent iteration of super/sub-buckets</p>
- *
- * <p>Thread safety: safe for threads to hold their own iterators (no shared state),
- * as long as they also hold the ProgressToken object associated with it. No two
- * VisitorIterator instances may share the same progress token instance at the
- * same time.
- * Concurrent access to a single VisitorIterator instance is not safe and must
- * be handled atomically by the caller.</p>
- *
- * @author <a href="mailto:vekterli@yahoo-inc.com">Tor Brede Vekterli</a>
- */
-public class VisitorIterator {
- private ProgressToken progressToken;
- private BucketSource bucketSource;
- private int distributionBitCount;
-
- private static final Logger log = Logger.getLogger(VisitorIterator.class.getName());
-
- public static class BucketProgress {
- private BucketId superbucket;
- private BucketId progress;
-
- public BucketProgress(BucketId superbucket, BucketId progress) {
- this.superbucket = superbucket;
- this.progress = progress;
- }
-
- public BucketId getProgress() {
- return progress;
- }
-
- public BucketId getSuperbucket() {
- return superbucket;
- }
- }
-
- /**
- * Provides an abstract interface to <code>VisitorIterator</code> for
- * how pending buckets are acquired, decoupling this from the iteration
- * itself.
- *
- * <em>Important</em>: it is the responsibility of the {@link BucketSource} implementation
- * to ensure that progress information is honored for (partially) finished buckets.
- * From the point of view of the iterator itself, it should not have to deal with
- * filtering away already finished buckets, as this is a detail best left to
- * bucket sources.
- */
- protected static interface BucketSource {
- public boolean hasNext();
- public boolean shouldYield();
- public boolean visitsAllBuckets();
- public BucketProgress getNext();
- public long getTotalBucketCount();
- public int getDistributionBitCount();
- public void setDistributionBitCount(int distributionBitCount,
- ProgressToken progress);
- public void update(BucketId superbucket, BucketId progress,
- ProgressToken token);
- }
-
- /**
- * Provides a bucket source that encompasses the entire range available
- * through a given value of distribution bits
- */
- protected static class DistributionRangeBucketSource implements BucketSource {
- private boolean flushActive = false;
- private int distributionBitCount;
- // Wouldn't need this if this were a non-static class, but do it for
- // the sake of keeping things identical in Java and C++
- private ProgressToken progressToken;
-
- public DistributionRangeBucketSource(int distributionBitCount,
- ProgressToken progress) {
- progressToken = progress;
-
- // New progress token (could also be empty, in which this is a
- // no-op anyway)
- if (progressToken.getTotalBucketCount() == 0) {
- assert(progressToken.isEmpty()) : "inconsistent progress state";
- progressToken.setTotalBucketCount(1L << distributionBitCount);
- progressToken.setDistributionBitCount(distributionBitCount);
- progressToken.setBucketCursor(0);
- progressToken.setFinishedBucketCount(0);
- this.distributionBitCount = distributionBitCount;
- }
- else {
- this.distributionBitCount = progressToken.getDistributionBitCount();
- // Quick consistency check to ensure the user isn't trying to eg.
- // pass a progress token for an explicit document selection
- if (progressToken.getTotalBucketCount() != (1L << progressToken.getDistributionBitCount())) {
- throw new IllegalArgumentException("Total bucket count in existing progress is not "
- + "consistent with that of the current document selection");
- }
- }
-
- if (!progress.isFinished()) {
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Importing unfinished progress token with " +
- "bits: " + progressToken.getDistributionBitCount() +
- ", active: " + progressToken.getActiveBucketCount() +
- ", pending: " + progressToken.getPendingBucketCount() +
- ", cursor: " + progressToken.getBucketCursor() +
- ", finished: " + progressToken.getFinishedBucketCount() +
- ", total: " + progressToken.getTotalBucketCount());
- }
- if (!progress.isEmpty()) {
- // Lower all active to pending
- if (progressToken.getActiveBucketCount() > 0) {
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Progress token had active buckets upon range " +
- "construction. Setting these as pending");
- }
- progressToken.setAllBucketsToState(ProgressToken.BucketState.BUCKET_PENDING);
- }
- // Fixup for any buckets that were active when progress was written
- // but are now pending and with wrong dist bits (used-bits). Buckets
- // split here may very well be split/merged again if we set a new dist
- // bit count, but that is the desired process
- correctInconsistentPending(progressToken.getDistributionBitCount());
- // Fixup for bucket cursor in case of bucket space downscaling
- correctTruncatedBucketCursor();
-
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Partial bucket space progress; continuing "+
- "from position " + progressToken.getBucketCursor());
- }
- }
- progressToken.setFinishedBucketCount(progressToken.getBucketCursor() -
- progressToken.getPendingBucketCount());
- } else {
- assert(progressToken.getBucketCursor() == progressToken.getTotalBucketCount());
- }
- // Should be all fixed up and good to go
- progressToken.setInconsistentState(false);
- }
-
- protected boolean isLosslessResetPossible() {
- // #pending must be equal to cursor, i.e. all buckets ever fetched
- // must be located in the set of pending
- if (progressToken.getPendingBucketCount() != progressToken.getBucketCursor()) {
- return false;
- }
- // Check if all pending buckets have a progress of 0
- for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
- : progressToken.getBuckets().entrySet()) {
- if (entry.getValue().getState() != ProgressToken.BucketState.BUCKET_PENDING) {
- return false;
- }
- if (entry.getValue().getProgress().getId() != 0) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Ensure that a given <code>ProgressToken</code> instance only has
- * buckets pending that have a used-bits count of that of the
- * <code>targetDistCits</code>. This is done by splitting or merging
- * all inconsistent buckets until the desired state is reached.
- *
- * Time complexity is approx <i>O(4bn)</i> where <i>b</i> is the maximum
- * delta of bits to change anywhere in the set of pending and <i>n</i>
- * is the number of pending. This includes the time spent making shallow
- * map copies.
- *
- * @param targetDistBits The desired distribution bit count of the buckets
- */
- private void correctInconsistentPending(int targetDistBits) {
- boolean maybeInconsistent = true;
- long bucketsSplit = 0, bucketsMerged = 0;
- long pendingBefore = progressToken.getPendingBucketCount();
- ProgressToken p = progressToken;
-
- // Optimization: before doing any splitting/merging at all, we check
- // to see if we can't simply just reset the entire internal state
- // with the new distribution bit count. This ensures that if we go
- // from eg. 1 bit to 20 bits, we won't have to perform a grueling
- // half a million splits to cover the same bucket space as that 1
- // single-bit bucket once did
- if (isLosslessResetPossible()) {
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "At start of bucket space and all " +
- "buckets have no progress; doing a lossless reset " +
- "instead of splitting/merging");
- }
- assert(p.getActiveBucketCount() == 0);
- p.clearAllBuckets();
- p.setBucketCursor(0);
- return;
- }
-
- while (maybeInconsistent) {
- BucketId lastMergedBucket = null;
- maybeInconsistent = false;
- // Make a shallow working copy of the bucket map. BucketKeyWrapper
- // keys are considered immutable, and should thus not be at risk
- // for being changed during the inner loop
- // Do separate passes for splitting and merging just to make
- // absolutely sure that the two ops won't step on each others'
- // toes. This isn't wildly efficient, but the data sets in question
- // are presumed to be low in size and this is presumed to be a very
- // infrequent operation
- TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> buckets
- = new TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry>(p.getBuckets());
- for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
- : buckets.entrySet()) {
- assert(entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING);
- BucketId pending = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey()));
- if (pending.getUsedBits() < targetDistBits) {
- if (pending.getUsedBits() + 1 < targetDistBits) {
- maybeInconsistent = true; // Do another pass
- }
- p.splitPendingBucket(pending);
- ++bucketsSplit;
- }
- }
-
- // Make new map copy with potentially split buckets
- buckets = new TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry>(p.getBuckets());
- for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
- : buckets.entrySet()) {
- assert(entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING);
- BucketId pending = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey()));
- if (pending.getUsedBits() > targetDistBits) {
- // If this is the right sibling of an already merged left sibling,
- // it's already been merged away, so we should skip it
- if (lastMergedBucket != null) {
- BucketId rightCheck = new BucketId(lastMergedBucket.getUsedBits(),
- lastMergedBucket.getId() | (1L << (lastMergedBucket.getUsedBits() - 1)));
- if (pending.equals(rightCheck)) {
- if (log.isLoggable(LogLevel.SPAM)) {
- log.log(LogLevel.SPAM, "Skipped " + pending +
- ", as it was right sibling of " + lastMergedBucket);
- }
- continue;
- }
- }
- if (pending.getUsedBits() - 1 > targetDistBits) {
- maybeInconsistent = true; // Do another pass
- }
- p.mergePendingBucket(pending);
- ++bucketsMerged;
-
- lastMergedBucket = pending;
- }
- }
- }
- if ((bucketsSplit > 0 || bucketsMerged > 0) && log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Existing progress' pending buckets had inconsistent " +
- "distribution bits; performed " + bucketsSplit + " split ops and " +
- bucketsMerged + " merge ops. Pending: " + pendingBefore + " -> " +
- p.getPendingBucketCount());
- }
- }
-
- private void correctTruncatedBucketCursor() {
- // We've truncated the bucket cursor, but in doing so we might
- // have moved back beyond where there are pending buckets. Consider
- // having a cursor value of 3 at 31 bits and then moving to 11 bits.
- // With 1 pending we'll normally reach a cursor of 0, even though it
- // should be 1
- for (ProgressToken.BucketKeyWrapper bucketKey
- : progressToken.getBuckets().keySet()) {
- BucketId bid = bucketKey.toBucketId();
- long idx = bucketKey.getKey() >>> (64 - bid.getUsedBits());
- if (bid.getUsedBits() == distributionBitCount
- && idx >= progressToken.getBucketCursor()) {
- progressToken.setBucketCursor(idx + 1);
- }
- }
- if (log.isLoggable(LogLevel.SPAM)) {
- log.log(LogLevel.SPAM, "New range bucket cursor is " +
- progressToken.getBucketCursor());
- }
- }
-
- public boolean hasNext() {
- return progressToken.getBucketCursor() < (1L << distributionBitCount);
- }
-
- public boolean shouldYield() {
- // If we need to flush all active buckets, stall the iteration until
- // this has been done
- return flushActive;
- }
-
- public boolean visitsAllBuckets() {
- return true;
- }
-
- public long getTotalBucketCount() {
- return 1L << distributionBitCount;
- }
-
- public BucketProgress getNext() {
- assert(hasNext()) : "getNext() called with hasNext() == false";
- long currentPosition = progressToken.getBucketCursor();
- long key = ProgressToken.makeNthBucketKey(currentPosition, distributionBitCount);
- ++currentPosition;
- progressToken.setBucketCursor(currentPosition);
- return new BucketProgress(
- new BucketId(ProgressToken.keyToBucketId(key)),
- new BucketId());
- }
-
- public int getDistributionBitCount() {
- return distributionBitCount;
- }
-
- public void setDistributionBitCount(int distributionBitCount,
- ProgressToken progress)
- {
- this.distributionBitCount = distributionBitCount;
-
- // There might be a case where we're waiting for active buckets
- // already when a new distribution bit change comes in. If so,
- // don't do anything at all yet with the set of pending
- if (progressToken.getActiveBucketCount() > 0) {
- flushActive = true;
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Holding off new/pending buckets and consistency " +
- "correction until all " + progress.getActiveBucketCount() +
- " active buckets have been updated");
- }
- progressToken.setInconsistentState(true);
- } else {
- // Only perform the actual distribution bit bucket ops if we've
- // got no pending buckets
- int delta = distributionBitCount - progressToken.getDistributionBitCount();
-
- // Must do this before setting the bucket cursor to allow
- // reset-checking to be performed
- correctInconsistentPending(distributionBitCount);
- if (delta > 0) {
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Increasing distribution bits for full bucket " +
- "space range source from " + progressToken.getDistributionBitCount() + " to " +
- distributionBitCount);
- }
- progressToken.setFinishedBucketCount(progressToken.getFinishedBucketCount() << delta);
- // By n-doubling the position, the bucket key ordering ensures
- // we go from eg. 3:0x02 to 4:0x02 to 5:02 etc.
- progressToken.setBucketCursor(progressToken.getBucketCursor() << delta);
- } else if (delta < 0) {
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Decreasing distribution bits for full bucket " +
- "space range source from " + progressToken.getDistributionBitCount() +
- " to " + distributionBitCount + " bits");
- }
- // Scale down bucket space and cursor
- progressToken.setBucketCursor(progressToken.getBucketCursor() >>> -delta);
- progressToken.setFinishedBucketCount(progressToken.getFinishedBucketCount() >>> -delta);
- }
-
- progressToken.setTotalBucketCount(1L << distributionBitCount);
- progressToken.setDistributionBitCount(distributionBitCount);
-
- correctTruncatedBucketCursor();
- progressToken.setInconsistentState(false);
- }
- }
-
- public void update(BucketId superbucket, BucketId progress,
- ProgressToken token) {
- progressToken.updateProgress(superbucket, progress);
-
- if (superbucket.getUsedBits() != distributionBitCount) {
- if (!progress.equals(ProgressToken.FINISHED_BUCKET)) {
- // We should now always flush active buckets before doing a
- // consistency fix. This simplifies things greatly
- assert(flushActive);
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Received non-finished bucket " +
- superbucket + " with wrong distribution bit count (" +
- superbucket.getUsedBits() + "). Waiting to correct " +
- "until all active are done");
- }
- } else {
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Received finished bucket " +
- superbucket + " with wrong distribution bit count (" +
- superbucket.getUsedBits() + "). Waiting to correct " +
- "until all active are done");
- }
- }
- }
-
- if (progressToken.getActiveBucketCount() == 0) {
- if (flushActive) {
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "All active buckets flushed, " +
- "correcting progress token and continuing normal operation");
- }
- // Trigger the actual bucket state change this time
- setDistributionBitCount(distributionBitCount, progressToken);
- assert(progressToken.getDistributionBitCount() == distributionBitCount);
- }
- flushActive = false;
- // Update #finished since we might have had inconsistent active
- // buckets that have prevent us from getting a correct value. At
- // this point, however, all pending buckets should presumably be
- // at the same, correct dist bit count, so we can safely compute
- // a new count
- // TODO: ensure this is consistent
- if (progressToken.getPendingBucketCount() <= progressToken.getBucketCursor()) {
- progressToken.setFinishedBucketCount(progressToken.getBucketCursor() -
- progressToken.getPendingBucketCount());
- }
- }
- }
- }
-
- /**
- * Provides an explicit set of bucket IDs to iterate over. Will immediately
- * set these as pending in the {@link ProgressToken}, as it is presumed this set is
- * rather small. Changing the distribution bit count for this source is
- * effectively a no-op, as explicit bucket IDs should not be implicitly
- * changed.
- */
- protected static class ExplicitBucketSource implements BucketSource {
- private int distributionBitCount;
- private long totalBucketCount = 0;
-
- public ExplicitBucketSource(Set<BucketId> superbuckets,
- int distributionBitCount,
- ProgressToken progress) {
- this.distributionBitCount = progress.getDistributionBitCount();
- this.totalBucketCount = superbuckets.size();
-
- // New progress token?
- if (progress.getTotalBucketCount() == 0) {
- progress.setTotalBucketCount(this.totalBucketCount);
- progress.setDistributionBitCount(distributionBitCount);
- this.distributionBitCount = distributionBitCount;
- }
- else {
- // Quick consistency check to ensure the user isn't trying to eg.
- // pass a progress token for another document selection
- if (progress.getTotalBucketCount() != totalBucketCount
- || (progress.getFinishedBucketCount() + progress.getPendingBucketCount()
- + progress.getActiveBucketCount() != totalBucketCount)) {
- throw new IllegalArgumentException("Total bucket count in existing progress is not " +
- "consistent with that of the current document selection");
- }
- if (progress.getBucketCursor() != 0) {
- // Trying to use a range source progress file
- throw new IllegalArgumentException("Cannot use given progress file with the "+
- "current document selection");
- }
- this.distributionBitCount = progress.getDistributionBitCount();
- }
-
- if (progress.isFinished() || !progress.isEmpty()) return;
-
- for (BucketId id : superbuckets) {
- // Add all superbuckets with zero sub-bucket progress and pending
- progress.addBucket(id, new BucketId(), ProgressToken.BucketState.BUCKET_PENDING);
- }
- }
-
- public boolean hasNext() {
- return false;
- }
-
- public boolean shouldYield() {
- return false;
- }
-
- public boolean visitsAllBuckets() {
- return false;
- }
-
- public long getTotalBucketCount() {
- return totalBucketCount;
- }
-
- // All explicit buckets should have been placed in the progress
- // token during construction, so this method should never be called
- public BucketProgress getNext() {
- throw new IllegalStateException("getNext() called on ExplicitBucketSource");
- }
-
- public int getDistributionBitCount() {
- return distributionBitCount;
- }
-
- public void setDistributionBitCount(int distributionBitCount,
- ProgressToken progress)
- {
- // Setting distribution bits for explicit bucket source is essentially
- // a no-op, since its buckets already are fixed at 32 used bits.
- progress.setDistributionBitCount(distributionBitCount);
- this.distributionBitCount = distributionBitCount;
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Set distribution bit count to "
- + distributionBitCount + " for explicit bucket source (no-op)");
- }
- }
-
- public void update(BucketId superbucket, BucketId progress,
- ProgressToken token) {
- // Simply delegate to ProgressToken, as it maintains all progress state
- token.updateProgress(superbucket, progress);
- }
- }
-
- /**
- * @param bucketSource An instance of {@link BucketSource}, providing the working set for
- * the iterator
- * @param progressToken A {@link ProgressToken} instance, allowing the progress of
- * finished or partially finished buckets to be tracked
- *
- * @see BucketSource
- * @see ProgressToken
- */
- private VisitorIterator(ProgressToken progressToken,
- BucketSource bucketSource) {
- assert(progressToken.getDistributionBitCount() == bucketSource.getDistributionBitCount())
- : "inconsistent distribution bit counts";
- this.distributionBitCount = progressToken.getDistributionBitCount();
- this.progressToken = progressToken;
- this.bucketSource = bucketSource;
- }
-
-
- /**
- * @return The pair [superbucket, progress] that specifies the next iterable
- * bucket. When a superbucket is initially returned, the pair is equal to
- * that of [superbucket, 0], as there has been no progress into its sub-buckets
- * yet (if they exist).
- *
- * Precondition: <code>hasNext() == true</code>
- */
- public BucketProgress getNext() {
- assert(progressToken.getDistributionBitCount() == bucketSource.getDistributionBitCount())
- : "inconsistent distribution bit counts for progress and source";
- assert(hasNext());
- // We prioritize returning buckets in the pending map over those
- // that may be in the bucket source, since we want to avoid growing
- // the map too much
- if (progressToken.hasPending()) {
- // Find first pending bucket in token
- TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> buckets = progressToken.getBuckets();
- ProgressToken.BucketEntry pending = null;
- BucketId superbucket = null;
- for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry : buckets.entrySet()) {
- if (entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING) {
- pending = entry.getValue();
- superbucket = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey()));
- break;
- }
- }
- assert(pending != null) : "getNext() called with inconsistent state";
-
- // Set bucket to active, since it's not awaiting an update
- pending.setState(ProgressToken.BucketState.BUCKET_ACTIVE);
-
- progressToken.setActiveBucketCount(progressToken.getActiveBucketCount() + 1);
- progressToken.setPendingBucketCount(progressToken.getPendingBucketCount() - 1);
-
- return new BucketProgress(superbucket, pending.getProgress());
- } else {
- BucketProgress ret = bucketSource.getNext();
- progressToken.addBucket(ret.getSuperbucket(), ret.getProgress(),
- ProgressToken.BucketState.BUCKET_ACTIVE);
- return ret;
- }
- }
-
- /**
- * <p>Check whether or not it is valid to call {@link #getNext()} with the current
- * iterator state.</p>
- *
- * <p>There exists a case wherein <code>hasNext</code> may return false before {@link #update} is
- * called, but true afterwards. This happens when the set of pending buckets is
- * empty, the bucket source is empty <em>but</em> the set of active buckets is
- * not. A future progress update on any of the buckets in the active set may
- * or may not make that bucket available to the pending set again.
- * This must be handled explicitly by the caller by checking {@link #isDone()}
- * and ensuring that {@link #update} is called before retrying <code>hasNext</code>.</p>
- *
- * <p>This method will also return false if the number of distribution bits have
- * changed and there are active buckets needing to be flushed before the
- * iterator will allow new buckets to be handed out.</p>
- *
- * @return Whether or not it is valid to call {@link #getNext()} with the current
- * iterator state.
- */
- public boolean hasNext() {
- return (progressToken.hasPending() || bucketSource.hasNext()) && !bucketSource.shouldYield();
- }
-
- /**
- * Check if the iterator is actually done
- *
- * @see #hasNext()
- *
- * @return <code>true</code> <em>iff</em> the bucket source is empty and
- * there are no pending or active buckets in the progress token.
- */
- public boolean isDone() {
- return !(hasNext() || progressToken.hasActive());
- }
-
- /**
- * <p>Tell the iterator that we've finished processing up to <i>and
- * including</i> <code>progress</code>. <code>progress</code> may be a sub-bucket <i>or</i>
- * the invalid 0-bucket (in case the caller fails to process the bucket and
- * must return it to the set of pending) <em>or</em> the special case <code>BucketId(Integer.MAX_VALUE)</code>,
- * the latter indicating to the iterator that traversal is complete for
- * <code>superbucket</code>'s tree. The null bucket should only be used if no
- * non-null updates have yet been given for the superbucket.</p>
- *
- * <p>It is a requirement that each superbucket returned by {@link #getNext()} must
- * eventually result in 1-n update operations, where the last update operation
- * has the special progress==super case.</p>
- *
- * <p>If the document selection used to create the iterator is unknown and there
- * were active buckets at the time of a distribution bit state change, such
- * a bucket passed to <code>update()</code> will be in an inconsistent state
- * with regards to the number of bits it uses. For unfinished buckets, this
- * is handled by splitting or merging it until it's consistent, depending on
- * whether or not it had a lower or higher distribution bit count than that of
- * the current system state. For finished buckets of a lower dist bit count,
- * the amount of finished buckets in the ProgressToken is adjusted upwards
- * to compensate for the fact that a bucket using fewer distribution bits
- * actually covers more of the bucket space than the ones that are currently
- * in use. For finished buckets of a higher dist bit count, the number of
- * finished buckets is <em>not</em> increased at that point in time, since
- * such a bucket doesn't actually cover an entire bucket with the current state.</p>
- *
- * <p>All this is done automatically and transparently to the caller once all
- * active buckets have been updated.</p>
- *
- * @param superbucket A valid bucket ID that has been retrieved earlier through
- * {@link #getNext()}
- * @param progress A bucket logically contained within <code>super</code>. Subsequent
- * updates for the same superbucket must have <code>progress</code> be in an increasing
- * order, where order is defined as the in-order traversal of the bucket split
- * tree. May also be the null bucket if the superbucket has not seen any "proper"
- * progress updates yet or the special case Integer.MAX_VALUE. Note that inconsistent
- * splitting might actually see <code>progress</code> as containing <code>super</code>
- * rather than vice versa, so this is explicitly allowed to pass by the code.
- */
- public void update(BucketId superbucket, BucketId progress) {
- // Delegate to bucket source, as it knows how to deal with buckets
- // that are in an inconsistent state wrt distribution bit count
- bucketSource.update(superbucket, progress, progressToken);
- }
-
- /**
- * @return The total number of iterable buckets that remain to be processed
- *
- * Note: currently includes all non-finished (i.e. active and pending
- * buckets) as well
- */
- public long getRemainingBucketCount() {
- return progressToken.getTotalBucketCount() - progressToken.getFinishedBucketCount();
- }
-
- /**
- * @return Internal bucket source instance. Do <i>NOT</i> modify!
- */
- protected BucketSource getBucketSource() {
- return bucketSource;
- }
-
- public ProgressToken getProgressToken() {
- return progressToken;
- }
-
- public int getDistributionBitCount() {
- return distributionBitCount;
- }
-
- /**
- * <p>Set the distribution bit count for the iterator and the buckets it
- * currently maintains and will return in the future.</p>
- *
- * <p>For document selections that result in a explicit set of buckets, this
- * is essentially a no-op, so in such a case, disregard the rest of this text.</p>
- *
- * <p>Changing the number of distribution bits for an unknown document
- * selection will effectively scale the bucket space that will be visited;
- * each bit increase or decrease doubling or halving its size, respectively.
- * When increasing, any pending buckets will be split to ensure the total
- * bucket space covered remains the same. Correspondingly, when decreasing,
- * any pending buckets will be merged appropriately.</p>
- *
- * <p>If there are buckets active at the time of the change, the actual
- * bucket splitting/merging operations are kept on hold until all active
- * buckets have been updated, at which point they will be automatically
- * performed. The iterator will force such an update by not giving out
- * any new or pending buckets until that happens.</p>
- *
- * <p><em>Note:</em> when decreasing the number of distribution bits,
- * there is a chance of losing superbucket progress in a bucket that
- * is merged with another bucket, leading to potential duplicate
- * results.</p>
- *
- * @param distBits New system state distribution bit count
- */
- public void setDistributionBitCount(int distBits) {
- if (distributionBitCount != distBits) {
- bucketSource.setDistributionBitCount(distBits, progressToken);
- distributionBitCount = distBits;
- if (log.isLoggable(LogLevel.DEBUG)) {
- log.log(LogLevel.DEBUG, "Set visitor iterator distribution bit count to "
- + distBits);
- }
- }
- }
-
- public boolean visitsAllBuckets() {
- return bucketSource.visitsAllBuckets();
- }
-
- /**
- * Create a new <code>VisitorIterator</code> instance based on the given document
- * selection string.
- *
- * @param documentSelection Document selection string used to create the
- * <code>VisitorIterator</code> instance. Depending on the characteristics of the
- * selection, the iterator may iterate over only a small subset of the buckets or
- * every bucket in the system. Both cases will be handled efficiently.
- * @param idFactory {@link BucketId} factory specifying the number of distribution bits
- * to use et al.
- * @param progress A unique {@link ProgressToken} instance which is used for maintaining the state
- * of the iterator. Can <em>not</em> be shared with other iterator instances at the same time.
- * If <code>progress</code> contains work done in an earlier iteration run, the iterator will pick
- * up from where it left off
- * @return A new <code>VisitorIterator</code> instance
- * @throws ParseException if <code>documentSelection</code> fails to properly parse
- */
- public static VisitorIterator createFromDocumentSelection(
- String documentSelection,
- BucketIdFactory idFactory,
- int distributionBitCount,
- ProgressToken progress) throws ParseException {
- BucketSelector bucketSel = new BucketSelector(idFactory);
- Set<BucketId> rawBuckets = bucketSel.getBucketList(documentSelection);
- BucketSource src;
-
- // Depending on whether the expression yielded an unknown number of
- // buckets, we create either an explicit bucket source or a distribution
- // bit-based range source
- if (rawBuckets == null) {
- // Range source
- src = new DistributionRangeBucketSource(distributionBitCount, progress);
- } else {
- // Explicit source
- src = new ExplicitBucketSource(rawBuckets, distributionBitCount, progress);
- }
-
- return new VisitorIterator(progress, src);
- }
-
- /**
- * Create a new <code>VisitorIterator</code> instance based on the given
- * set of buckets. This is supported for internal use only, and is required
- * by Synchronization. Use {@link #createFromDocumentSelection} instead for
- * all normal purposes.
- *
- * @param bucketsToVisit The set of buckets that will be visited
- * @param distributionBitCount Number of distribution bits to use
- * @param progress A unique ProgressToken instance which is used for maintaining the state
- * of the iterator. Can <em>not</em> be shared with other iterator instances at the same time.
- * If <code>progress</code> contains work done in an earlier iteration run, the iterator will pick
- * up from where it left off
- * @return A new <code>VisitorIterator</code> instance
- */
- public static VisitorIterator createFromExplicitBucketSet(
- Set<BucketId> bucketsToVisit,
- int distributionBitCount,
- ProgressToken progress) {
- // For obvious reasons, always create an explicit source here
- BucketSource src = new ExplicitBucketSource(bucketsToVisit,
- distributionBitCount, progress);
- return new VisitorIterator(progress, src);
- }
-}
+package com.yahoo.documentapi; + +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.BucketSelector; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.log.LogLevel; + +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.logging.Logger; + +/** + * <p>Enables transparent iteration of super/sub-buckets</p> + * + * <p>Thread safety: safe for threads to hold their own iterators (no shared state), + * as long as they also hold the ProgressToken object associated with it. No two + * VisitorIterator instances may share the same progress token instance at the + * same time. + * Concurrent access to a single VisitorIterator instance is not safe and must + * be handled atomically by the caller.</p> + * + * @author <a href="mailto:vekterli@yahoo-inc.com">Tor Brede Vekterli</a> + */ +public class VisitorIterator { + private ProgressToken progressToken; + private BucketSource bucketSource; + private int distributionBitCount; + + private static final Logger log = Logger.getLogger(VisitorIterator.class.getName()); + + public static class BucketProgress { + private BucketId superbucket; + private BucketId progress; + + public BucketProgress(BucketId superbucket, BucketId progress) { + this.superbucket = superbucket; + this.progress = progress; + } + + public BucketId getProgress() { + return progress; + } + + public BucketId getSuperbucket() { + return superbucket; + } + } + + /** + * Provides an abstract interface to <code>VisitorIterator</code> for + * how pending buckets are acquired, decoupling this from the iteration + * itself. + * + * <em>Important</em>: it is the responsibility of the {@link BucketSource} implementation + * to ensure that progress information is honored for (partially) finished buckets. + * From the point of view of the iterator itself, it should not have to deal with + * filtering away already finished buckets, as this is a detail best left to + * bucket sources. + */ + protected static interface BucketSource { + public boolean hasNext(); + public boolean shouldYield(); + public boolean visitsAllBuckets(); + public BucketProgress getNext(); + public long getTotalBucketCount(); + public int getDistributionBitCount(); + public void setDistributionBitCount(int distributionBitCount, + ProgressToken progress); + public void update(BucketId superbucket, BucketId progress, + ProgressToken token); + } + + /** + * Provides a bucket source that encompasses the entire range available + * through a given value of distribution bits + */ + protected static class DistributionRangeBucketSource implements BucketSource { + private boolean flushActive = false; + private int distributionBitCount; + // Wouldn't need this if this were a non-static class, but do it for + // the sake of keeping things identical in Java and C++ + private ProgressToken progressToken; + + public DistributionRangeBucketSource(int distributionBitCount, + ProgressToken progress) { + progressToken = progress; + + // New progress token (could also be empty, in which this is a + // no-op anyway) + if (progressToken.getTotalBucketCount() == 0) { + assert(progressToken.isEmpty()) : "inconsistent progress state"; + progressToken.setTotalBucketCount(1L << distributionBitCount); + progressToken.setDistributionBitCount(distributionBitCount); + progressToken.setBucketCursor(0); + progressToken.setFinishedBucketCount(0); + this.distributionBitCount = distributionBitCount; + } + else { + this.distributionBitCount = progressToken.getDistributionBitCount(); + // Quick consistency check to ensure the user isn't trying to eg. + // pass a progress token for an explicit document selection + if (progressToken.getTotalBucketCount() != (1L << progressToken.getDistributionBitCount())) { + throw new IllegalArgumentException("Total bucket count in existing progress is not " + + "consistent with that of the current document selection"); + } + } + + if (!progress.isFinished()) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Importing unfinished progress token with " + + "bits: " + progressToken.getDistributionBitCount() + + ", active: " + progressToken.getActiveBucketCount() + + ", pending: " + progressToken.getPendingBucketCount() + + ", cursor: " + progressToken.getBucketCursor() + + ", finished: " + progressToken.getFinishedBucketCount() + + ", total: " + progressToken.getTotalBucketCount()); + } + if (!progress.isEmpty()) { + // Lower all active to pending + if (progressToken.getActiveBucketCount() > 0) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Progress token had active buckets upon range " + + "construction. Setting these as pending"); + } + progressToken.setAllBucketsToState(ProgressToken.BucketState.BUCKET_PENDING); + } + // Fixup for any buckets that were active when progress was written + // but are now pending and with wrong dist bits (used-bits). Buckets + // split here may very well be split/merged again if we set a new dist + // bit count, but that is the desired process + correctInconsistentPending(progressToken.getDistributionBitCount()); + // Fixup for bucket cursor in case of bucket space downscaling + correctTruncatedBucketCursor(); + + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Partial bucket space progress; continuing "+ + "from position " + progressToken.getBucketCursor()); + } + } + progressToken.setFinishedBucketCount(progressToken.getBucketCursor() - + progressToken.getPendingBucketCount()); + } else { + assert(progressToken.getBucketCursor() == progressToken.getTotalBucketCount()); + } + // Should be all fixed up and good to go + progressToken.setInconsistentState(false); + } + + protected boolean isLosslessResetPossible() { + // #pending must be equal to cursor, i.e. all buckets ever fetched + // must be located in the set of pending + if (progressToken.getPendingBucketCount() != progressToken.getBucketCursor()) { + return false; + } + // Check if all pending buckets have a progress of 0 + for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry + : progressToken.getBuckets().entrySet()) { + if (entry.getValue().getState() != ProgressToken.BucketState.BUCKET_PENDING) { + return false; + } + if (entry.getValue().getProgress().getId() != 0) { + return false; + } + } + return true; + } + + /** + * Ensure that a given <code>ProgressToken</code> instance only has + * buckets pending that have a used-bits count of that of the + * <code>targetDistCits</code>. This is done by splitting or merging + * all inconsistent buckets until the desired state is reached. + * + * Time complexity is approx <i>O(4bn)</i> where <i>b</i> is the maximum + * delta of bits to change anywhere in the set of pending and <i>n</i> + * is the number of pending. This includes the time spent making shallow + * map copies. + * + * @param targetDistBits The desired distribution bit count of the buckets + */ + private void correctInconsistentPending(int targetDistBits) { + boolean maybeInconsistent = true; + long bucketsSplit = 0, bucketsMerged = 0; + long pendingBefore = progressToken.getPendingBucketCount(); + ProgressToken p = progressToken; + + // Optimization: before doing any splitting/merging at all, we check + // to see if we can't simply just reset the entire internal state + // with the new distribution bit count. This ensures that if we go + // from eg. 1 bit to 20 bits, we won't have to perform a grueling + // half a million splits to cover the same bucket space as that 1 + // single-bit bucket once did + if (isLosslessResetPossible()) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "At start of bucket space and all " + + "buckets have no progress; doing a lossless reset " + + "instead of splitting/merging"); + } + assert(p.getActiveBucketCount() == 0); + p.clearAllBuckets(); + p.setBucketCursor(0); + return; + } + + while (maybeInconsistent) { + BucketId lastMergedBucket = null; + maybeInconsistent = false; + // Make a shallow working copy of the bucket map. BucketKeyWrapper + // keys are considered immutable, and should thus not be at risk + // for being changed during the inner loop + // Do separate passes for splitting and merging just to make + // absolutely sure that the two ops won't step on each others' + // toes. This isn't wildly efficient, but the data sets in question + // are presumed to be low in size and this is presumed to be a very + // infrequent operation + TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> buckets + = new TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry>(p.getBuckets()); + for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry + : buckets.entrySet()) { + assert(entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING); + BucketId pending = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey())); + if (pending.getUsedBits() < targetDistBits) { + if (pending.getUsedBits() + 1 < targetDistBits) { + maybeInconsistent = true; // Do another pass + } + p.splitPendingBucket(pending); + ++bucketsSplit; + } + } + + // Make new map copy with potentially split buckets + buckets = new TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry>(p.getBuckets()); + for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry + : buckets.entrySet()) { + assert(entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING); + BucketId pending = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey())); + if (pending.getUsedBits() > targetDistBits) { + // If this is the right sibling of an already merged left sibling, + // it's already been merged away, so we should skip it + if (lastMergedBucket != null) { + BucketId rightCheck = new BucketId(lastMergedBucket.getUsedBits(), + lastMergedBucket.getId() | (1L << (lastMergedBucket.getUsedBits() - 1))); + if (pending.equals(rightCheck)) { + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "Skipped " + pending + + ", as it was right sibling of " + lastMergedBucket); + } + continue; + } + } + if (pending.getUsedBits() - 1 > targetDistBits) { + maybeInconsistent = true; // Do another pass + } + p.mergePendingBucket(pending); + ++bucketsMerged; + + lastMergedBucket = pending; + } + } + } + if ((bucketsSplit > 0 || bucketsMerged > 0) && log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Existing progress' pending buckets had inconsistent " + + "distribution bits; performed " + bucketsSplit + " split ops and " + + bucketsMerged + " merge ops. Pending: " + pendingBefore + " -> " + + p.getPendingBucketCount()); + } + } + + private void correctTruncatedBucketCursor() { + // We've truncated the bucket cursor, but in doing so we might + // have moved back beyond where there are pending buckets. Consider + // having a cursor value of 3 at 31 bits and then moving to 11 bits. + // With 1 pending we'll normally reach a cursor of 0, even though it + // should be 1 + for (ProgressToken.BucketKeyWrapper bucketKey + : progressToken.getBuckets().keySet()) { + BucketId bid = bucketKey.toBucketId(); + long idx = bucketKey.getKey() >>> (64 - bid.getUsedBits()); + if (bid.getUsedBits() == distributionBitCount + && idx >= progressToken.getBucketCursor()) { + progressToken.setBucketCursor(idx + 1); + } + } + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "New range bucket cursor is " + + progressToken.getBucketCursor()); + } + } + + public boolean hasNext() { + return progressToken.getBucketCursor() < (1L << distributionBitCount); + } + + public boolean shouldYield() { + // If we need to flush all active buckets, stall the iteration until + // this has been done + return flushActive; + } + + public boolean visitsAllBuckets() { + return true; + } + + public long getTotalBucketCount() { + return 1L << distributionBitCount; + } + + public BucketProgress getNext() { + assert(hasNext()) : "getNext() called with hasNext() == false"; + long currentPosition = progressToken.getBucketCursor(); + long key = ProgressToken.makeNthBucketKey(currentPosition, distributionBitCount); + ++currentPosition; + progressToken.setBucketCursor(currentPosition); + return new BucketProgress( + new BucketId(ProgressToken.keyToBucketId(key)), + new BucketId()); + } + + public int getDistributionBitCount() { + return distributionBitCount; + } + + public void setDistributionBitCount(int distributionBitCount, + ProgressToken progress) + { + this.distributionBitCount = distributionBitCount; + + // There might be a case where we're waiting for active buckets + // already when a new distribution bit change comes in. If so, + // don't do anything at all yet with the set of pending + if (progressToken.getActiveBucketCount() > 0) { + flushActive = true; + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Holding off new/pending buckets and consistency " + + "correction until all " + progress.getActiveBucketCount() + + " active buckets have been updated"); + } + progressToken.setInconsistentState(true); + } else { + // Only perform the actual distribution bit bucket ops if we've + // got no pending buckets + int delta = distributionBitCount - progressToken.getDistributionBitCount(); + + // Must do this before setting the bucket cursor to allow + // reset-checking to be performed + correctInconsistentPending(distributionBitCount); + if (delta > 0) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Increasing distribution bits for full bucket " + + "space range source from " + progressToken.getDistributionBitCount() + " to " + + distributionBitCount); + } + progressToken.setFinishedBucketCount(progressToken.getFinishedBucketCount() << delta); + // By n-doubling the position, the bucket key ordering ensures + // we go from eg. 3:0x02 to 4:0x02 to 5:02 etc. + progressToken.setBucketCursor(progressToken.getBucketCursor() << delta); + } else if (delta < 0) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Decreasing distribution bits for full bucket " + + "space range source from " + progressToken.getDistributionBitCount() + + " to " + distributionBitCount + " bits"); + } + // Scale down bucket space and cursor + progressToken.setBucketCursor(progressToken.getBucketCursor() >>> -delta); + progressToken.setFinishedBucketCount(progressToken.getFinishedBucketCount() >>> -delta); + } + + progressToken.setTotalBucketCount(1L << distributionBitCount); + progressToken.setDistributionBitCount(distributionBitCount); + + correctTruncatedBucketCursor(); + progressToken.setInconsistentState(false); + } + } + + public void update(BucketId superbucket, BucketId progress, + ProgressToken token) { + progressToken.updateProgress(superbucket, progress); + + if (superbucket.getUsedBits() != distributionBitCount) { + if (!progress.equals(ProgressToken.FINISHED_BUCKET)) { + // We should now always flush active buckets before doing a + // consistency fix. This simplifies things greatly + assert(flushActive); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Received non-finished bucket " + + superbucket + " with wrong distribution bit count (" + + superbucket.getUsedBits() + "). Waiting to correct " + + "until all active are done"); + } + } else { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Received finished bucket " + + superbucket + " with wrong distribution bit count (" + + superbucket.getUsedBits() + "). Waiting to correct " + + "until all active are done"); + } + } + } + + if (progressToken.getActiveBucketCount() == 0) { + if (flushActive) { + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "All active buckets flushed, " + + "correcting progress token and continuing normal operation"); + } + // Trigger the actual bucket state change this time + setDistributionBitCount(distributionBitCount, progressToken); + assert(progressToken.getDistributionBitCount() == distributionBitCount); + } + flushActive = false; + // Update #finished since we might have had inconsistent active + // buckets that have prevent us from getting a correct value. At + // this point, however, all pending buckets should presumably be + // at the same, correct dist bit count, so we can safely compute + // a new count + // TODO: ensure this is consistent + if (progressToken.getPendingBucketCount() <= progressToken.getBucketCursor()) { + progressToken.setFinishedBucketCount(progressToken.getBucketCursor() - + progressToken.getPendingBucketCount()); + } + } + } + } + + /** + * Provides an explicit set of bucket IDs to iterate over. Will immediately + * set these as pending in the {@link ProgressToken}, as it is presumed this set is + * rather small. Changing the distribution bit count for this source is + * effectively a no-op, as explicit bucket IDs should not be implicitly + * changed. + */ + protected static class ExplicitBucketSource implements BucketSource { + private int distributionBitCount; + private long totalBucketCount = 0; + + public ExplicitBucketSource(Set<BucketId> superbuckets, + int distributionBitCount, + ProgressToken progress) { + this.distributionBitCount = progress.getDistributionBitCount(); + this.totalBucketCount = superbuckets.size(); + + // New progress token? + if (progress.getTotalBucketCount() == 0) { + progress.setTotalBucketCount(this.totalBucketCount); + progress.setDistributionBitCount(distributionBitCount); + this.distributionBitCount = distributionBitCount; + } + else { + // Quick consistency check to ensure the user isn't trying to eg. + // pass a progress token for another document selection + if (progress.getTotalBucketCount() != totalBucketCount + || (progress.getFinishedBucketCount() + progress.getPendingBucketCount() + + progress.getActiveBucketCount() != totalBucketCount)) { + throw new IllegalArgumentException("Total bucket count in existing progress is not " + + "consistent with that of the current document selection"); + } + if (progress.getBucketCursor() != 0) { + // Trying to use a range source progress file + throw new IllegalArgumentException("Cannot use given progress file with the "+ + "current document selection"); + } + this.distributionBitCount = progress.getDistributionBitCount(); + } + + if (progress.isFinished() || !progress.isEmpty()) return; + + for (BucketId id : superbuckets) { + // Add all superbuckets with zero sub-bucket progress and pending + progress.addBucket(id, new BucketId(), ProgressToken.BucketState.BUCKET_PENDING); + } + } + + public boolean hasNext() { + return false; + } + + public boolean shouldYield() { + return false; + } + + public boolean visitsAllBuckets() { + return false; + } + + public long getTotalBucketCount() { + return totalBucketCount; + } + + // All explicit buckets should have been placed in the progress + // token during construction, so this method should never be called + public BucketProgress getNext() { + throw new IllegalStateException("getNext() called on ExplicitBucketSource"); + } + + public int getDistributionBitCount() { + return distributionBitCount; + } + + public void setDistributionBitCount(int distributionBitCount, + ProgressToken progress) + { + // Setting distribution bits for explicit bucket source is essentially + // a no-op, since its buckets already are fixed at 32 used bits. + progress.setDistributionBitCount(distributionBitCount); + this.distributionBitCount = distributionBitCount; + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Set distribution bit count to " + + distributionBitCount + " for explicit bucket source (no-op)"); + } + } + + public void update(BucketId superbucket, BucketId progress, + ProgressToken token) { + // Simply delegate to ProgressToken, as it maintains all progress state + token.updateProgress(superbucket, progress); + } + } + + /** + * @param bucketSource An instance of {@link BucketSource}, providing the working set for + * the iterator + * @param progressToken A {@link ProgressToken} instance, allowing the progress of + * finished or partially finished buckets to be tracked + * + * @see BucketSource + * @see ProgressToken + */ + private VisitorIterator(ProgressToken progressToken, + BucketSource bucketSource) { + assert(progressToken.getDistributionBitCount() == bucketSource.getDistributionBitCount()) + : "inconsistent distribution bit counts"; + this.distributionBitCount = progressToken.getDistributionBitCount(); + this.progressToken = progressToken; + this.bucketSource = bucketSource; + } + + + /** + * @return The pair [superbucket, progress] that specifies the next iterable + * bucket. When a superbucket is initially returned, the pair is equal to + * that of [superbucket, 0], as there has been no progress into its sub-buckets + * yet (if they exist). + * + * Precondition: <code>hasNext() == true</code> + */ + public BucketProgress getNext() { + assert(progressToken.getDistributionBitCount() == bucketSource.getDistributionBitCount()) + : "inconsistent distribution bit counts for progress and source"; + assert(hasNext()); + // We prioritize returning buckets in the pending map over those + // that may be in the bucket source, since we want to avoid growing + // the map too much + if (progressToken.hasPending()) { + // Find first pending bucket in token + TreeMap<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> buckets = progressToken.getBuckets(); + ProgressToken.BucketEntry pending = null; + BucketId superbucket = null; + for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry : buckets.entrySet()) { + if (entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING) { + pending = entry.getValue(); + superbucket = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey())); + break; + } + } + assert(pending != null) : "getNext() called with inconsistent state"; + + // Set bucket to active, since it's not awaiting an update + pending.setState(ProgressToken.BucketState.BUCKET_ACTIVE); + + progressToken.setActiveBucketCount(progressToken.getActiveBucketCount() + 1); + progressToken.setPendingBucketCount(progressToken.getPendingBucketCount() - 1); + + return new BucketProgress(superbucket, pending.getProgress()); + } else { + BucketProgress ret = bucketSource.getNext(); + progressToken.addBucket(ret.getSuperbucket(), ret.getProgress(), + ProgressToken.BucketState.BUCKET_ACTIVE); + return ret; + } + } + + /** + * <p>Check whether or not it is valid to call {@link #getNext()} with the current + * iterator state.</p> + * + * <p>There exists a case wherein <code>hasNext</code> may return false before {@link #update} is + * called, but true afterwards. This happens when the set of pending buckets is + * empty, the bucket source is empty <em>but</em> the set of active buckets is + * not. A future progress update on any of the buckets in the active set may + * or may not make that bucket available to the pending set again. + * This must be handled explicitly by the caller by checking {@link #isDone()} + * and ensuring that {@link #update} is called before retrying <code>hasNext</code>.</p> + * + * <p>This method will also return false if the number of distribution bits have + * changed and there are active buckets needing to be flushed before the + * iterator will allow new buckets to be handed out.</p> + * + * @return Whether or not it is valid to call {@link #getNext()} with the current + * iterator state. + */ + public boolean hasNext() { + return (progressToken.hasPending() || bucketSource.hasNext()) && !bucketSource.shouldYield(); + } + + /** + * Check if the iterator is actually done + * + * @see #hasNext() + * + * @return <code>true</code> <em>iff</em> the bucket source is empty and + * there are no pending or active buckets in the progress token. + */ + public boolean isDone() { + return !(hasNext() || progressToken.hasActive()); + } + + /** + * <p>Tell the iterator that we've finished processing up to <i>and + * including</i> <code>progress</code>. <code>progress</code> may be a sub-bucket <i>or</i> + * the invalid 0-bucket (in case the caller fails to process the bucket and + * must return it to the set of pending) <em>or</em> the special case <code>BucketId(Integer.MAX_VALUE)</code>, + * the latter indicating to the iterator that traversal is complete for + * <code>superbucket</code>'s tree. The null bucket should only be used if no + * non-null updates have yet been given for the superbucket.</p> + * + * <p>It is a requirement that each superbucket returned by {@link #getNext()} must + * eventually result in 1-n update operations, where the last update operation + * has the special progress==super case.</p> + * + * <p>If the document selection used to create the iterator is unknown and there + * were active buckets at the time of a distribution bit state change, such + * a bucket passed to <code>update()</code> will be in an inconsistent state + * with regards to the number of bits it uses. For unfinished buckets, this + * is handled by splitting or merging it until it's consistent, depending on + * whether or not it had a lower or higher distribution bit count than that of + * the current system state. For finished buckets of a lower dist bit count, + * the amount of finished buckets in the ProgressToken is adjusted upwards + * to compensate for the fact that a bucket using fewer distribution bits + * actually covers more of the bucket space than the ones that are currently + * in use. For finished buckets of a higher dist bit count, the number of + * finished buckets is <em>not</em> increased at that point in time, since + * such a bucket doesn't actually cover an entire bucket with the current state.</p> + * + * <p>All this is done automatically and transparently to the caller once all + * active buckets have been updated.</p> + * + * @param superbucket A valid bucket ID that has been retrieved earlier through + * {@link #getNext()} + * @param progress A bucket logically contained within <code>super</code>. Subsequent + * updates for the same superbucket must have <code>progress</code> be in an increasing + * order, where order is defined as the in-order traversal of the bucket split + * tree. May also be the null bucket if the superbucket has not seen any "proper" + * progress updates yet or the special case Integer.MAX_VALUE. Note that inconsistent + * splitting might actually see <code>progress</code> as containing <code>super</code> + * rather than vice versa, so this is explicitly allowed to pass by the code. + */ + public void update(BucketId superbucket, BucketId progress) { + // Delegate to bucket source, as it knows how to deal with buckets + // that are in an inconsistent state wrt distribution bit count + bucketSource.update(superbucket, progress, progressToken); + } + + /** + * @return The total number of iterable buckets that remain to be processed + * + * Note: currently includes all non-finished (i.e. active and pending + * buckets) as well + */ + public long getRemainingBucketCount() { + return progressToken.getTotalBucketCount() - progressToken.getFinishedBucketCount(); + } + + /** + * @return Internal bucket source instance. Do <i>NOT</i> modify! + */ + protected BucketSource getBucketSource() { + return bucketSource; + } + + public ProgressToken getProgressToken() { + return progressToken; + } + + public int getDistributionBitCount() { + return distributionBitCount; + } + + /** + * <p>Set the distribution bit count for the iterator and the buckets it + * currently maintains and will return in the future.</p> + * + * <p>For document selections that result in a explicit set of buckets, this + * is essentially a no-op, so in such a case, disregard the rest of this text.</p> + * + * <p>Changing the number of distribution bits for an unknown document + * selection will effectively scale the bucket space that will be visited; + * each bit increase or decrease doubling or halving its size, respectively. + * When increasing, any pending buckets will be split to ensure the total + * bucket space covered remains the same. Correspondingly, when decreasing, + * any pending buckets will be merged appropriately.</p> + * + * <p>If there are buckets active at the time of the change, the actual + * bucket splitting/merging operations are kept on hold until all active + * buckets have been updated, at which point they will be automatically + * performed. The iterator will force such an update by not giving out + * any new or pending buckets until that happens.</p> + * + * <p><em>Note:</em> when decreasing the number of distribution bits, + * there is a chance of losing superbucket progress in a bucket that + * is merged with another bucket, leading to potential duplicate + * results.</p> + * + * @param distBits New system state distribution bit count + */ + public void setDistributionBitCount(int distBits) { + if (distributionBitCount != distBits) { + bucketSource.setDistributionBitCount(distBits, progressToken); + distributionBitCount = distBits; + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Set visitor iterator distribution bit count to " + + distBits); + } + } + } + + public boolean visitsAllBuckets() { + return bucketSource.visitsAllBuckets(); + } + + /** + * Create a new <code>VisitorIterator</code> instance based on the given document + * selection string. + * + * @param documentSelection Document selection string used to create the + * <code>VisitorIterator</code> instance. Depending on the characteristics of the + * selection, the iterator may iterate over only a small subset of the buckets or + * every bucket in the system. Both cases will be handled efficiently. + * @param idFactory {@link BucketId} factory specifying the number of distribution bits + * to use et al. + * @param progress A unique {@link ProgressToken} instance which is used for maintaining the state + * of the iterator. Can <em>not</em> be shared with other iterator instances at the same time. + * If <code>progress</code> contains work done in an earlier iteration run, the iterator will pick + * up from where it left off + * @return A new <code>VisitorIterator</code> instance + * @throws ParseException if <code>documentSelection</code> fails to properly parse + */ + public static VisitorIterator createFromDocumentSelection( + String documentSelection, + BucketIdFactory idFactory, + int distributionBitCount, + ProgressToken progress) throws ParseException { + BucketSelector bucketSel = new BucketSelector(idFactory); + Set<BucketId> rawBuckets = bucketSel.getBucketList(documentSelection); + BucketSource src; + + // Depending on whether the expression yielded an unknown number of + // buckets, we create either an explicit bucket source or a distribution + // bit-based range source + if (rawBuckets == null) { + // Range source + src = new DistributionRangeBucketSource(distributionBitCount, progress); + } else { + // Explicit source + src = new ExplicitBucketSource(rawBuckets, distributionBitCount, progress); + } + + return new VisitorIterator(progress, src); + } + + /** + * Create a new <code>VisitorIterator</code> instance based on the given + * set of buckets. This is supported for internal use only, and is required + * by Synchronization. Use {@link #createFromDocumentSelection} instead for + * all normal purposes. + * + * @param bucketsToVisit The set of buckets that will be visited + * @param distributionBitCount Number of distribution bits to use + * @param progress A unique ProgressToken instance which is used for maintaining the state + * of the iterator. Can <em>not</em> be shared with other iterator instances at the same time. + * If <code>progress</code> contains work done in an earlier iteration run, the iterator will pick + * up from where it left off + * @return A new <code>VisitorIterator</code> instance + */ + public static VisitorIterator createFromExplicitBucketSet( + Set<BucketId> bucketsToVisit, + int distributionBitCount, + ProgressToken progress) { + // For obvious reasons, always create an explicit source here + BucketSource src = new ExplicitBucketSource(bucketsToVisit, + distributionBitCount, progress); + return new VisitorIterator(progress, src); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java index 966caa46969..aa56f69b966 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java @@ -1,103 +1,103 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.local;
-
-import com.yahoo.document.Document;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.DocumentPut;
-import com.yahoo.document.DocumentRemove;
-import com.yahoo.document.DocumentType;
-import com.yahoo.document.DocumentUpdate;
-import com.yahoo.document.TestAndSetCondition;
-import com.yahoo.documentapi.Response;
-import com.yahoo.documentapi.Result;
-import com.yahoo.documentapi.SyncSession;
-import com.yahoo.documentapi.messagebus.protocol.*;
-
-/**
- * @author bratseth
- */
-public class LocalSyncSession implements SyncSession {
-
- private LocalDocumentAccess access;
-
- public LocalSyncSession(LocalDocumentAccess access) {
- this.access = access;
- }
-
- @Override
- public void put(DocumentPut documentPut) {
- if (documentPut.getCondition().isPresent()) {
- throw new UnsupportedOperationException("test-and-set is not supported.");
- }
-
- access.documents.put(documentPut.getId(), documentPut.getDocument());
- }
-
- @Override
- public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) {
- access.documents.put(documentPut.getId(), documentPut.getDocument());
- }
-
- @Override
- public Document get(DocumentId id) {
- return access.documents.get(id);
- }
-
- @Override
- public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) {
- // FIXME: More than half the get() methods are deprecated, but they all
- // call exactly the same method, including this one, throwing away most
- // of the parameters
- return access.documents.get(id);
- }
-
- @Override
- public boolean remove(DocumentRemove documentRemove) {
- if (documentRemove.getCondition().isPresent()) {
- throw new UnsupportedOperationException("test-and-set is not supported.");
- }
- access.documents.remove(documentRemove.getId());
- return true;
- }
-
- @Override
- public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority) {
- return remove(documentRemove);
- }
-
- @Override
- public boolean update(DocumentUpdate update) {
- Document document = access.documents.get(update.getId());
- if (document == null) {
- return false;
- }
- update.applyTo(document);
- return true;
- }
-
- @Override
- public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) {
- Document document = access.documents.get(update.getId());
- if (document == null) {
- return false;
- }
- update.applyTo(document);
- return true;
- }
-
- @Override
- public Response getNext() {
- throw new UnsupportedOperationException("Queue not supported.");
- }
-
- @Override
- public Response getNext(int timeout) {
- throw new UnsupportedOperationException("Queue not supported.");
- }
-
- @Override
- public void destroy() {
- access = null;
- }
-
-}
+package com.yahoo.documentapi.local; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.documentapi.Response; +import com.yahoo.documentapi.Result; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.messagebus.protocol.*; + +/** + * @author bratseth + */ +public class LocalSyncSession implements SyncSession { + + private LocalDocumentAccess access; + + public LocalSyncSession(LocalDocumentAccess access) { + this.access = access; + } + + @Override + public void put(DocumentPut documentPut) { + if (documentPut.getCondition().isPresent()) { + throw new UnsupportedOperationException("test-and-set is not supported."); + } + + access.documents.put(documentPut.getId(), documentPut.getDocument()); + } + + @Override + public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { + access.documents.put(documentPut.getId(), documentPut.getDocument()); + } + + @Override + public Document get(DocumentId id) { + return access.documents.get(id); + } + + @Override + public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) { + // FIXME: More than half the get() methods are deprecated, but they all + // call exactly the same method, including this one, throwing away most + // of the parameters + return access.documents.get(id); + } + + @Override + public boolean remove(DocumentRemove documentRemove) { + if (documentRemove.getCondition().isPresent()) { + throw new UnsupportedOperationException("test-and-set is not supported."); + } + access.documents.remove(documentRemove.getId()); + return true; + } + + @Override + public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority) { + return remove(documentRemove); + } + + @Override + public boolean update(DocumentUpdate update) { + Document document = access.documents.get(update.getId()); + if (document == null) { + return false; + } + update.applyTo(document); + return true; + } + + @Override + public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) { + Document document = access.documents.get(update.getId()); + if (document == null) { + return false; + } + update.applyTo(document); + return true; + } + + @Override + public Response getNext() { + throw new UnsupportedOperationException("Queue not supported."); + } + + @Override + public Response getNext(int timeout) { + throw new UnsupportedOperationException("Queue not supported."); + } + + @Override + public void destroy() { + access = null; + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index 392913e8b36..0d4061c5f25 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -26,11 +26,11 @@ import java.util.concurrent.ScheduledExecutorService; public class MessageBusDocumentAccess extends DocumentAccess { private final NetworkMessageBus bus; - + private final MessageBusParams params; // TODO: make pool size configurable? ScheduledExecutorService is not dynamic - private final ScheduledExecutorService scheduledExecutorService = - Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors(), + private final ScheduledExecutorService scheduledExecutorService = + Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors(), ThreadFactoryFactory.getDaemonThreadFactory("mbus.access.scheduler")); /** @@ -63,7 +63,7 @@ public class MessageBusDocumentAccess extends DocumentAccess { throw new DocumentAccessException(e); } } - + private MessageBus messageBus() { return bus.getMessageBus(); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSession.java index 7f5544a93ce..92af9a3758b 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSession.java @@ -1,38 +1,38 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus;
-
-/**
- * This class defines a common interface for message bus sessions.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public interface MessageBusSession {
-
- /**
- * Returns the route to send all messages to when sending through this session.
- *
- * @return The route string.
- */
- public String getRoute();
-
- /**
- * Sets the route to send all messages to when sending through this session.
- *
- * @param route The route string.
- */
- public void setRoute(String route);
-
- /**
- * Returns the trace level used when sending messages through this session.
- *
- * @return The trace level.
- */
- public int getTraceLevel();
-
- /**
- * Sets the trace level used when sending messages through this session.
- *
- * @param traceLevel The trace level to set.
- */
- public void setTraceLevel(int traceLevel);
-}
+package com.yahoo.documentapi.messagebus; + +/** + * This class defines a common interface for message bus sessions. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface MessageBusSession { + + /** + * Returns the route to send all messages to when sending through this session. + * + * @return The route string. + */ + public String getRoute(); + + /** + * Sets the route to send all messages to when sending through this session. + * + * @param route The route string. + */ + public void setRoute(String route); + + /** + * Returns the trace level used when sending messages through this session. + * + * @return The trace level. + */ + public int getTraceLevel(); + + /** + * Sets the trace level used when sending messages through this session. + * + * @param traceLevel The trace level to set. + */ + public void setTraceLevel(int traceLevel); +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java index 5be94564556..095d0c14a49 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -1,225 +1,225 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus;
-
-import com.yahoo.document.Document;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.DocumentPut;
-import com.yahoo.document.DocumentRemove;
-import com.yahoo.document.DocumentUpdate;
-import com.yahoo.documentapi.AsyncParameters;
-import com.yahoo.documentapi.DocumentAccessException;
-import com.yahoo.documentapi.Response;
-import com.yahoo.documentapi.Result;
-import com.yahoo.documentapi.SyncParameters;
-import com.yahoo.documentapi.SyncSession;
-import com.yahoo.documentapi.messagebus.protocol.*;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.messagebus.Message;
-import com.yahoo.messagebus.MessageBus;
-import com.yahoo.messagebus.Reply;
-import com.yahoo.messagebus.ReplyHandler;
-
-/**
- * An implementation of the SyncSession interface running over message bus.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class MessageBusSyncSession implements MessageBusSession, SyncSession, ReplyHandler {
-
- private MessageBusAsyncSession session;
-
- /**
- * Creates a new sync session running on message bus logic.
- *
- * @param syncParams Common syncsession parameters, not used.
- * @param bus The message bus on which to run.
- * @param mbusParams Parameters concerning message bus configuration.
- */
- MessageBusSyncSession(SyncParameters syncParams, MessageBus bus, MessageBusParams mbusParams) {
- session = new MessageBusAsyncSession(new AsyncParameters(), bus, mbusParams, this);
- }
-
- @Override
- public void handleReply(Reply reply) {
- if (reply.getContext() instanceof RequestMonitor) {
- ((RequestMonitor)reply.getContext()).replied(reply);
- } else {
- ReplyHandler handler = reply.getCallStack().pop(reply);
- handler.handleReply(reply); // not there yet
- }
- }
-
- @Override
- public Response getNext() {
- throw new UnsupportedOperationException("Queue not supported.");
- }
-
- @Override
- public Response getNext(int timeout) {
- throw new UnsupportedOperationException("Queue not supported.");
- }
-
- @Override
- public void destroy() {
- session.destroy();
- }
-
- /**
- * Perform a synchronous sending of a message. This method block until the message is successfuly sent and a
- * corresponding reply has been received.
- *
- * @param msg The message to send.
- * @return The reply received.
- */
- public Reply syncSend(Message msg) {
- try {
- RequestMonitor monitor = new RequestMonitor();
- msg.setContext(monitor);
- msg.pushHandler(this); // store monitor
- Result result = null;
- while (result == null || result.getType() == Result.ResultType.TRANSIENT_ERROR) {
- result = session.send(msg);
- if (result != null && result.isSuccess()) {
- break;
- }
- Thread.sleep(100);
- }
- if (!result.isSuccess()) {
- throw new DocumentAccessException(result.getError().toString());
- }
- return monitor.waitForReply();
- } catch (InterruptedException e) {
- throw new DocumentAccessException(e);
- }
- }
-
- @Override
- public void put(DocumentPut documentPut) {
- put(documentPut, DocumentProtocol.Priority.NORMAL_3);
- }
-
- @Override
- public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) {
- PutDocumentMessage msg = new PutDocumentMessage(documentPut);
- msg.setPriority(priority);
- syncSendPutDocumentMessage(msg);
- }
-
- @Override
- public Document get(DocumentId id) {
- return get(id, "[all]", DocumentProtocol.Priority.NORMAL_1);
- }
-
- @Override
- public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) {
- GetDocumentMessage msg = new GetDocumentMessage(id, fieldSet);
- msg.setPriority(pri);
-
- Reply reply = syncSend(msg);
- if (reply.hasErrors()) {
- throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply));
- }
- if (reply.getType() != DocumentProtocol.REPLY_GETDOCUMENT) {
- throw new DocumentAccessException("Received unknown response: " + reply);
- }
- GetDocumentReply docReply = ((GetDocumentReply)reply);
- Document doc = docReply.getDocument();
- if (doc != null) {
- doc.setLastModified(docReply.getLastModified());
- }
- return doc;
- }
-
- @Override
- public boolean remove(DocumentRemove documentRemove) {
- RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId());
- msg.setCondition(documentRemove.getCondition());
- return remove(msg);
- }
-
- @Override
- public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority pri) {
- RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId());
- msg.setPriority(pri);
- msg.setCondition(documentRemove.getCondition());
- return remove(msg);
- }
-
- private boolean remove(RemoveDocumentMessage msg) {
- Reply reply = syncSend(msg);
- if (reply.hasErrors()) {
- throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply));
- }
- if (reply.getType() != DocumentProtocol.REPLY_REMOVEDOCUMENT) {
- throw new DocumentAccessException("Received unknown response: " + reply);
- }
- return ((RemoveDocumentReply)reply).wasFound();
- }
-
- @Override
- public boolean update(DocumentUpdate update) {
- return update(update, DocumentProtocol.Priority.NORMAL_2);
- }
-
- @Override
- public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) {
- UpdateDocumentMessage msg = new UpdateDocumentMessage(update);
- msg.setPriority(pri);
- Reply reply = syncSend(msg);
- if (reply.hasErrors()) {
- throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply),
- MessageBusAsyncSession.getErrorCodes(reply));
- }
- if (reply.getType() != DocumentProtocol.REPLY_UPDATEDOCUMENT) {
- throw new DocumentAccessException("Received unknown response: " + reply);
- }
- return ((UpdateDocumentReply)reply).wasFound();
- }
-
- @Override
- public String getRoute() {
- return session.getRoute();
- }
-
- @Override
- public void setRoute(String route) {
- session.setRoute(route);
- }
-
- @Override
- public int getTraceLevel() {
- return session.getTraceLevel();
- }
-
- @Override
- public void setTraceLevel(int traceLevel) {
- session.setTraceLevel(traceLevel);
- }
-
- /**
- * This class implements a monitor for waiting for a reply to arrive.
- */
- static class RequestMonitor {
- private Reply reply = null;
-
- synchronized Reply waitForReply() throws InterruptedException {
- while (reply == null) {
- wait();
- }
- return reply;
- }
-
- synchronized void replied(Reply reply) {
- this.reply = reply;
- notify();
- }
- }
-
- private void syncSendPutDocumentMessage(PutDocumentMessage putDocumentMessage) {
- Reply reply = syncSend(putDocumentMessage);
- if (reply.hasErrors()) {
- throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply),
- MessageBusAsyncSession.getErrorCodes(reply));
- }
- }
-}
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.DocumentAccessException; +import com.yahoo.documentapi.Response; +import com.yahoo.documentapi.Result; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.SyncSession; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.ReplyHandler; + +/** + * An implementation of the SyncSession interface running over message bus. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class MessageBusSyncSession implements MessageBusSession, SyncSession, ReplyHandler { + + private MessageBusAsyncSession session; + + /** + * Creates a new sync session running on message bus logic. + * + * @param syncParams Common syncsession parameters, not used. + * @param bus The message bus on which to run. + * @param mbusParams Parameters concerning message bus configuration. + */ + MessageBusSyncSession(SyncParameters syncParams, MessageBus bus, MessageBusParams mbusParams) { + session = new MessageBusAsyncSession(new AsyncParameters(), bus, mbusParams, this); + } + + @Override + public void handleReply(Reply reply) { + if (reply.getContext() instanceof RequestMonitor) { + ((RequestMonitor)reply.getContext()).replied(reply); + } else { + ReplyHandler handler = reply.getCallStack().pop(reply); + handler.handleReply(reply); // not there yet + } + } + + @Override + public Response getNext() { + throw new UnsupportedOperationException("Queue not supported."); + } + + @Override + public Response getNext(int timeout) { + throw new UnsupportedOperationException("Queue not supported."); + } + + @Override + public void destroy() { + session.destroy(); + } + + /** + * Perform a synchronous sending of a message. This method block until the message is successfuly sent and a + * corresponding reply has been received. + * + * @param msg The message to send. + * @return The reply received. + */ + public Reply syncSend(Message msg) { + try { + RequestMonitor monitor = new RequestMonitor(); + msg.setContext(monitor); + msg.pushHandler(this); // store monitor + Result result = null; + while (result == null || result.getType() == Result.ResultType.TRANSIENT_ERROR) { + result = session.send(msg); + if (result != null && result.isSuccess()) { + break; + } + Thread.sleep(100); + } + if (!result.isSuccess()) { + throw new DocumentAccessException(result.getError().toString()); + } + return monitor.waitForReply(); + } catch (InterruptedException e) { + throw new DocumentAccessException(e); + } + } + + @Override + public void put(DocumentPut documentPut) { + put(documentPut, DocumentProtocol.Priority.NORMAL_3); + } + + @Override + public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { + PutDocumentMessage msg = new PutDocumentMessage(documentPut); + msg.setPriority(priority); + syncSendPutDocumentMessage(msg); + } + + @Override + public Document get(DocumentId id) { + return get(id, "[all]", DocumentProtocol.Priority.NORMAL_1); + } + + @Override + public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) { + GetDocumentMessage msg = new GetDocumentMessage(id, fieldSet); + msg.setPriority(pri); + + Reply reply = syncSend(msg); + if (reply.hasErrors()) { + throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply)); + } + if (reply.getType() != DocumentProtocol.REPLY_GETDOCUMENT) { + throw new DocumentAccessException("Received unknown response: " + reply); + } + GetDocumentReply docReply = ((GetDocumentReply)reply); + Document doc = docReply.getDocument(); + if (doc != null) { + doc.setLastModified(docReply.getLastModified()); + } + return doc; + } + + @Override + public boolean remove(DocumentRemove documentRemove) { + RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId()); + msg.setCondition(documentRemove.getCondition()); + return remove(msg); + } + + @Override + public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority pri) { + RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId()); + msg.setPriority(pri); + msg.setCondition(documentRemove.getCondition()); + return remove(msg); + } + + private boolean remove(RemoveDocumentMessage msg) { + Reply reply = syncSend(msg); + if (reply.hasErrors()) { + throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply)); + } + if (reply.getType() != DocumentProtocol.REPLY_REMOVEDOCUMENT) { + throw new DocumentAccessException("Received unknown response: " + reply); + } + return ((RemoveDocumentReply)reply).wasFound(); + } + + @Override + public boolean update(DocumentUpdate update) { + return update(update, DocumentProtocol.Priority.NORMAL_2); + } + + @Override + public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) { + UpdateDocumentMessage msg = new UpdateDocumentMessage(update); + msg.setPriority(pri); + Reply reply = syncSend(msg); + if (reply.hasErrors()) { + throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), + MessageBusAsyncSession.getErrorCodes(reply)); + } + if (reply.getType() != DocumentProtocol.REPLY_UPDATEDOCUMENT) { + throw new DocumentAccessException("Received unknown response: " + reply); + } + return ((UpdateDocumentReply)reply).wasFound(); + } + + @Override + public String getRoute() { + return session.getRoute(); + } + + @Override + public void setRoute(String route) { + session.setRoute(route); + } + + @Override + public int getTraceLevel() { + return session.getTraceLevel(); + } + + @Override + public void setTraceLevel(int traceLevel) { + session.setTraceLevel(traceLevel); + } + + /** + * This class implements a monitor for waiting for a reply to arrive. + */ + static class RequestMonitor { + private Reply reply = null; + + synchronized Reply waitForReply() throws InterruptedException { + while (reply == null) { + wait(); + } + return reply; + } + + synchronized void replied(Reply reply) { + this.reply = reply; + notify(); + } + } + + private void syncSendPutDocumentMessage(PutDocumentMessage putDocumentMessage) { + Reply reply = syncSend(putDocumentMessage); + if (reply.hasErrors()) { + throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), + MessageBusAsyncSession.getErrorCodes(reply)); + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ANDPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ANDPolicy.java index 04818f80672..a8b190108fa 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ANDPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ANDPolicy.java @@ -1,67 +1,67 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.messagebus.metrics.MetricSet;
-import com.yahoo.messagebus.routing.Hop;
-import com.yahoo.messagebus.routing.Route;
-import com.yahoo.messagebus.routing.RoutingContext;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * An AND policy is a routing policy that can be used to write simple routes that split a message between multiple other
- * destinations. It can either be configured in a routing config, which will then produce a policy that always selects
- * all configured recipients, or it can be configured using the policy parameter (i.e. a string following the name of
- * the policy). Note that configured recipients take precedence over recipients configured in the parameter string.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class ANDPolicy implements DocumentProtocolRoutingPolicy {
-
- // A list of hops that are to always be selected when select() is invoked.
- private final List<Hop> hops = new ArrayList<Hop>();
-
- /**
- * Constructs a new AND policy that requires all recipients to be ok for it to merge their replies to an ok reply.
- * I.e. all errors in all child replies are copied into the merged reply.
- *
- * @param param A string of recipients to select unless recipients have been configured.
- */
- public ANDPolicy(String param) {
- if (param == null || param.isEmpty()) {
- return;
- }
- Route route = Route.parse(param);
- for (int i = 0; i < route.getNumHops(); ++i) {
- hops.add(route.getHop(i));
- }
- }
-
- // Inherit doc from RoutingPolicy.
- public void select(RoutingContext context) {
- if (hops.isEmpty()) {
- context.addChildren(context.getAllRecipients());
- } else {
- for (Hop hop : hops) {
- Route route = new Route(context.getRoute());
- route.setHop(0, hop);
- context.addChild(route);
- }
- }
- context.setSelectOnRetry(false);
- context.addConsumableError(DocumentProtocol.ERROR_MESSAGE_IGNORED);
- }
-
- // Inherit doc from RoutingPolicy.
- public void merge(RoutingContext context) {
- DocumentProtocol.merge(context);
- }
-
- public void destroy() {
- }
-
- public MetricSet getMetrics() {
- return null;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.Hop; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingContext; + +import java.util.ArrayList; +import java.util.List; + +/** + * An AND policy is a routing policy that can be used to write simple routes that split a message between multiple other + * destinations. It can either be configured in a routing config, which will then produce a policy that always selects + * all configured recipients, or it can be configured using the policy parameter (i.e. a string following the name of + * the policy). Note that configured recipients take precedence over recipients configured in the parameter string. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ANDPolicy implements DocumentProtocolRoutingPolicy { + + // A list of hops that are to always be selected when select() is invoked. + private final List<Hop> hops = new ArrayList<Hop>(); + + /** + * Constructs a new AND policy that requires all recipients to be ok for it to merge their replies to an ok reply. + * I.e. all errors in all child replies are copied into the merged reply. + * + * @param param A string of recipients to select unless recipients have been configured. + */ + public ANDPolicy(String param) { + if (param == null || param.isEmpty()) { + return; + } + Route route = Route.parse(param); + for (int i = 0; i < route.getNumHops(); ++i) { + hops.add(route.getHop(i)); + } + } + + // Inherit doc from RoutingPolicy. + public void select(RoutingContext context) { + if (hops.isEmpty()) { + context.addChildren(context.getAllRecipients()); + } else { + for (Hop hop : hops) { + Route route = new Route(context.getRoute()); + route.setHop(0, hop); + context.addChild(route); + } + } + context.setSelectOnRetry(false); + context.addConsumableError(DocumentProtocol.ERROR_MESSAGE_IGNORED); + } + + // Inherit doc from RoutingPolicy. + public void merge(RoutingContext context) { + DocumentProtocol.merge(context); + } + + public void destroy() { + } + + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateReply.java index 48eb41fdb5c..11155dd76be 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateReply.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/BatchDocumentUpdateReply.java @@ -1,29 +1,29 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import java.util.ArrayList;
-
-public class BatchDocumentUpdateReply extends WriteDocumentReply {
-
- private ArrayList<Boolean> documentsNotFound = new ArrayList<Boolean>();
-
- /**
- * Constructs a new reply with no content.
- */
- public BatchDocumentUpdateReply() {
- super(DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE);
- }
-
- /**
- * If all documents to update are found, this vector will be empty. If
- * one or more documents are not found, this vector will have the size of
- * the initial number of updates, with entries set to true where the
- * corresponding update was not found.
- *
- * @return Vector containing indices of not found documents, or empty
- * if all documents were found
- */
- public ArrayList<Boolean> getDocumentsNotFound() {
- return documentsNotFound;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import java.util.ArrayList; + +public class BatchDocumentUpdateReply extends WriteDocumentReply { + + private ArrayList<Boolean> documentsNotFound = new ArrayList<Boolean>(); + + /** + * Constructs a new reply with no content. + */ + public BatchDocumentUpdateReply() { + super(DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE); + } + + /** + * If all documents to update are found, this vector will be empty. If + * one or more documents are not found, this vector will have the size of + * the initial number of updates, with entries set to true where the + * corresponding update was not found. + * + * @return Vector containing indices of not found documents, or empty + * if all documents were found + */ + public ArrayList<Boolean> getDocumentsNotFound() { + return documentsNotFound; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListEntry.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListEntry.java index 8de0cfd204c..90615bdf9ad 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListEntry.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListEntry.java @@ -1,47 +1,47 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.Document;
-import com.yahoo.document.serialization.DocumentDeserializer;
-import com.yahoo.vespa.objects.Serializer;
-
-public class DocumentListEntry {
-
- private Document doc;
- private long timestamp;
- private boolean removeEntry;
-
- public DocumentListEntry(Document doc, long timestamp, boolean removeEntry) {
- this.doc = doc;
- this.timestamp = timestamp;
- this.removeEntry = removeEntry;
- }
-
- public void serialize(Serializer buf) {
- buf.putLong(null, timestamp);
- doc.serialize(buf);
- buf.putByte(null, (byte)(removeEntry ? 1 : 0));
- }
-
- public static int getApproxSize() {
- return 60; // optimzation. approximation is sufficient
- }
-
- public DocumentListEntry(DocumentDeserializer buf) {
- timestamp = buf.getLong(null);
- doc = new Document(buf);
- removeEntry = buf.getByte(null) > 0;
- }
-
- public Document getDocument() {
- return doc;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public boolean isRemoveEntry() {
- return removeEntry;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.Document; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.vespa.objects.Serializer; + +public class DocumentListEntry { + + private Document doc; + private long timestamp; + private boolean removeEntry; + + public DocumentListEntry(Document doc, long timestamp, boolean removeEntry) { + this.doc = doc; + this.timestamp = timestamp; + this.removeEntry = removeEntry; + } + + public void serialize(Serializer buf) { + buf.putLong(null, timestamp); + doc.serialize(buf); + buf.putByte(null, (byte)(removeEntry ? 1 : 0)); + } + + public static int getApproxSize() { + return 60; // optimzation. approximation is sufficient + } + + public DocumentListEntry(DocumentDeserializer buf) { + timestamp = buf.getLong(null); + doc = new Document(buf); + removeEntry = buf.getByte(null) > 0; + } + + public Document getDocument() { + return doc; + } + + public long getTimestamp() { + return timestamp; + } + + public boolean isRemoveEntry() { + return removeEntry; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListMessage.java index 448c2820ec3..95b51f20877 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentListMessage.java @@ -1,54 +1,54 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.BucketId;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DocumentListMessage extends VisitorMessage {
-
- private BucketId bucket = new BucketId(16, 0);
- private final List<DocumentListEntry> entries = new ArrayList<DocumentListEntry>();
-
- public DocumentListMessage() {
- // empty
- }
-
- public DocumentListMessage(DocumentListMessage cmd) {
- bucket = cmd.bucket;
- entries.addAll(cmd.entries);
- }
-
- public BucketId getBucketId() {
- return bucket;
- }
-
- public void setBucketId(BucketId id) {
- bucket = id;
- }
-
- public List<DocumentListEntry> getDocuments() {
- return entries;
- }
-
- @Override
- public DocumentReply createReply() {
- return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST);
- }
-
- @Override
- public int getType() {
- return DocumentProtocol.MESSAGE_DOCUMENTLIST;
- }
-
- @Override
- public int getApproxSize() {
- return DocumentListEntry.getApproxSize() * entries.size();
- }
-
- @Override
- public String toString() {
- return "DocumentListMessage(" + entries.toString() + ")";
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; + +import java.util.ArrayList; +import java.util.List; + +public class DocumentListMessage extends VisitorMessage { + + private BucketId bucket = new BucketId(16, 0); + private final List<DocumentListEntry> entries = new ArrayList<DocumentListEntry>(); + + public DocumentListMessage() { + // empty + } + + public DocumentListMessage(DocumentListMessage cmd) { + bucket = cmd.bucket; + entries.addAll(cmd.entries); + } + + public BucketId getBucketId() { + return bucket; + } + + public void setBucketId(BucketId id) { + bucket = id; + } + + public List<DocumentListEntry> getDocuments() { + return entries; + } + + @Override + public DocumentReply createReply() { + return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_DOCUMENTLIST; + } + + @Override + public int getApproxSize() { + return DocumentListEntry.getApproxSize() * entries.size(); + } + + @Override + public String toString() { + return "DocumentListMessage(" + entries.toString() + ")"; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java index c4839c87f69..7949b15d955 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java @@ -1,85 +1,85 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.documentapi.messagebus.loadtypes.LoadType;
-import com.yahoo.messagebus.Message;
-import com.yahoo.messagebus.Routable;
-import com.yahoo.text.Utf8String;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public abstract class DocumentMessage extends Message {
-
- private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3;
- private LoadType loadType = LoadType.DEFAULT;
-
- /**
- * Constructs a new message with no content.
- */
- public DocumentMessage() {
- // empty
- }
-
- /**
- * Creates and returns a reply to this message.
- *
- * @return The created reply.
- */
- public abstract DocumentReply createReply();
-
- @Override
- public void swapState(Routable rhs) {
- super.swapState(rhs);
- if (rhs instanceof DocumentMessage) {
- DocumentMessage msg = (DocumentMessage)rhs;
-
- DocumentProtocol.Priority pri = this.priority;
- this.priority = msg.priority;
- msg.priority = pri;
-
- LoadType lt = this.loadType;
- this.loadType = msg.loadType;
- msg.loadType = lt;
- }
- }
-
- /**
- * Returns the priority tag for this message. This is an optional tag added for VDS that is not interpreted by the
- * document protocol.
- *
- * @return The priority.
- */
- public DocumentProtocol.Priority getPriority() { return priority; }
-
- /**
- * Sets the priority tag for this message.
- *
- * @param priority The priority to set.
- */
- public void setPriority(DocumentProtocol.Priority priority) {
- this.priority = priority;
- }
-
- public LoadType getLoadType() {
- return loadType;
- }
-
- public void setLoadType(LoadType loadType) {
- if (loadType != null) {
- this.loadType = loadType;
- } else {
- this.loadType = LoadType.DEFAULT;
- }
- }
-
- @Override
- public int getApproxSize() {
- return 4 + 1; // type + priority
- }
-
- @Override
- public Utf8String getProtocol() {
- return DocumentProtocol.NAME;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Routable; +import com.yahoo.text.Utf8String; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class DocumentMessage extends Message { + + private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3; + private LoadType loadType = LoadType.DEFAULT; + + /** + * Constructs a new message with no content. + */ + public DocumentMessage() { + // empty + } + + /** + * Creates and returns a reply to this message. + * + * @return The created reply. + */ + public abstract DocumentReply createReply(); + + @Override + public void swapState(Routable rhs) { + super.swapState(rhs); + if (rhs instanceof DocumentMessage) { + DocumentMessage msg = (DocumentMessage)rhs; + + DocumentProtocol.Priority pri = this.priority; + this.priority = msg.priority; + msg.priority = pri; + + LoadType lt = this.loadType; + this.loadType = msg.loadType; + msg.loadType = lt; + } + } + + /** + * Returns the priority tag for this message. This is an optional tag added for VDS that is not interpreted by the + * document protocol. + * + * @return The priority. + */ + public DocumentProtocol.Priority getPriority() { return priority; } + + /** + * Sets the priority tag for this message. + * + * @param priority The priority to set. + */ + public void setPriority(DocumentProtocol.Priority priority) { + this.priority = priority; + } + + public LoadType getLoadType() { + return loadType; + } + + public void setLoadType(LoadType loadType) { + if (loadType != null) { + this.loadType = loadType; + } else { + this.loadType = LoadType.DEFAULT; + } + } + + @Override + public int getApproxSize() { + return 4 + 1; // type + priority + } + + @Override + public Utf8String getProtocol() { + return DocumentProtocol.NAME; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentReply.java index 126d85c5703..e28b11646ab 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentReply.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentReply.java @@ -1,53 +1,53 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.messagebus.Reply;
-import com.yahoo.text.Utf8String;
-
-/**
- * This class implements a generic document protocol reply that can be reused by document messages that require no
- * special reply implementation while still allowing applications to distinguish between types.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class DocumentReply extends Reply {
-
- private final int type;
- private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3;
-
- /**
- * Constructs a new reply of given type.
- *
- * @param type The type code to assign to this.
- */
- public DocumentReply(int type) {
- this.type = type;
- }
-
- /**
- * Returns the priority tag for this message.
- * @return The priority.
- */
- public DocumentProtocol.Priority getPriority() {
- return priority;
- }
-
- /**
- * Sets the priority tag for this message.
- *
- * @param priority The priority to set.
- */
- public void setPriority(DocumentProtocol.Priority priority) {
- this.priority = priority;
- }
-
- @Override
- public Utf8String getProtocol() {
- return DocumentProtocol.NAME;
- }
-
- @Override
- public final int getType() {
- return type;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.messagebus.Reply; +import com.yahoo.text.Utf8String; + +/** + * This class implements a generic document protocol reply that can be reused by document messages that require no + * special reply implementation while still allowing applications to distinguish between types. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class DocumentReply extends Reply { + + private final int type; + private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3; + + /** + * Constructs a new reply of given type. + * + * @param type The type code to assign to this. + */ + public DocumentReply(int type) { + this.type = type; + } + + /** + * Returns the priority tag for this message. + * @return The priority. + */ + public DocumentProtocol.Priority getPriority() { + return priority; + } + + /** + * Sets the priority tag for this message. + * + * @param priority The priority to set. + */ + public void setPriority(DocumentProtocol.Priority priority) { + this.priority = priority; + } + + @Override + public Utf8String getProtocol() { + return DocumentProtocol.NAME; + } + + @Override + public final int getType() { + return type; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ErrorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ErrorPolicy.java index 0b7310ad2fb..1246f1e309a 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ErrorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ErrorPolicy.java @@ -1,44 +1,44 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.messagebus.EmptyReply;
-import com.yahoo.messagebus.metrics.MetricSet;
-import com.yahoo.messagebus.routing.RoutingContext;
-
-/**
- * This policy assigns an error supplied at constructor time to the routing context when {@link #select(RoutingContext)}
- * is invoked. This is useful for returning error states to the client instead of those auto-generated by mbus when a
- * routing policy can not be created.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class ErrorPolicy implements DocumentProtocolRoutingPolicy {
-
- private final String msg;
-
- /**
- * Creates a new policy that will assign an {@link EmptyReply} with the given error to all routing contexts that
- * invoke {@link #select(RoutingContext)}.
- *
- * @param msg The message of the error to assign.
- */
- public ErrorPolicy(String msg) {
- this.msg = msg;
- }
-
- public void select(RoutingContext ctx) {
- ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, msg);
- }
-
- public void merge(RoutingContext ctx) {
- throw new AssertionError("Routing should not pass terminated selection.");
- }
-
- public void destroy() {
- }
-
-
- public MetricSet getMetrics() {
- return null;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.RoutingContext; + +/** + * This policy assigns an error supplied at constructor time to the routing context when {@link #select(RoutingContext)} + * is invoked. This is useful for returning error states to the client instead of those auto-generated by mbus when a + * routing policy can not be created. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ErrorPolicy implements DocumentProtocolRoutingPolicy { + + private final String msg; + + /** + * Creates a new policy that will assign an {@link EmptyReply} with the given error to all routing contexts that + * invoke {@link #select(RoutingContext)}. + * + * @param msg The message of the error to assign. + */ + public ErrorPolicy(String msg) { + this.msg = msg; + } + + public void select(RoutingContext ctx) { + ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, msg); + } + + public void merge(RoutingContext ctx) { + throw new AssertionError("Routing should not pass terminated selection."); + } + + public void destroy() { + } + + + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java index a843102f466..90140f22d90 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java @@ -1,147 +1,147 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.jrt.Supervisor;
-import com.yahoo.jrt.Transport;
-import com.yahoo.jrt.slobrok.api.Mirror;
-import com.yahoo.jrt.slobrok.api.SlobrokList;
-import com.yahoo.messagebus.ErrorCode;
-import com.yahoo.messagebus.metrics.MetricSet;
-import com.yahoo.messagebus.routing.Hop;
-import com.yahoo.messagebus.routing.Route;
-import com.yahoo.messagebus.routing.RoutingContext;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * This policy implements the necessary logic to communicate with an external Vespa application and resolve its list of
- * recipients using that other application's slobrok servers.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class ExternPolicy implements DocumentProtocolRoutingPolicy {
-
- private Supervisor orb = null;
- private Mirror mirror = null;
- private String pattern = null;
- private String session = null;
- private final String error;
- private int offset = 0;
- private int generation = 0;
- private final List<Hop> recipients = new ArrayList<>();
-
- /**
- * Constructs a new instance of this policy. The argument given is the connection spec to the slobrok to use for
- * resolving recipients, as well as the pattern to use when querying. This constructor does _not_ wait for the
- * mirror to become ready.
- *
- * @param arg The slobrok connection spec.
- */
- public ExternPolicy(String arg) {
- if (arg == null || arg.length() == 0) {
- error = "Expected parameter, got empty string.";
- return;
- }
- String[] args = arg.split(";", 2);
- if (args.length != 2 || args[0].length() == 0 || args[1].length() == 0) {
- error = "Expected parameter on the form '<spec>;<pattern>', got '" + arg + "'.";
- return;
- }
- int pos = args[1].lastIndexOf('/');
- if (pos < 0) {
- error = "Expected pattern on the form '<service>/<session>', got '" + args[1] + "'.";
- return;
- }
- SlobrokList slobroks = new SlobrokList();
- slobroks.setup(args[0].split(","));
- pattern = args[1];
- session = pattern.substring(pos);
- orb = new Supervisor(new Transport());
- mirror = new Mirror(orb, slobroks);
- error = null;
- }
-
- /**
- * This is a safety mechanism to allow the constructor to fail and signal that it can not be used.
- *
- * @return The error string, or null if no error.
- */
- public String getError() {
- return error;
- }
-
- /**
- * Returns the slobrok mirror used by this policy to resolve external recipients.
- *
- * @return The external mirror.
- */
- public Mirror getMirror() {
- return mirror;
- }
-
- /**
- * Returns the appropriate recipient hop. This method provides synchronized access to the internal mirror.
- *
- * @return The recipient hop to use.
- */
- private synchronized Hop getRecipient() {
- update();
- if (recipients.isEmpty()) {
- return null;
- }
- int offset = ++this.offset & Integer.MAX_VALUE; // mask signed bit because of modulo
- return new Hop(recipients.get(offset % recipients.size()));
- }
-
- /**
- * Updates the list of matching recipients by querying the extern slobrok.
- */
- private void update() {
- int upd = mirror.updates();
- if (generation != upd) {
- generation = upd;
- recipients.clear();
- Mirror.Entry[] arr = mirror.lookup(pattern);
- for (Mirror.Entry entry : arr) {
- recipients.add(Hop.parse(entry.getSpec() + session));
- }
- }
- }
-
- @Override
- public void finalize() throws Throwable {
- super.finalize();
- mirror.shutdown();
- orb.transport().shutdown().join();
- }
-
- public void select(RoutingContext ctx) {
- if (error != null) {
- ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, error);
- } else if (mirror.ready()) {
- Hop hop = getRecipient();
- if (hop != null) {
- Route route = new Route(ctx.getRoute());
- route.setHop(0, hop);
- ctx.addChild(route);
- } else {
- ctx.setError(ErrorCode.NO_ADDRESS_FOR_SERVICE,
- "Could not resolve any recipients from '" + pattern + "'.");
- }
- } else {
- ctx.setError(ErrorCode.APP_TRANSIENT_ERROR, "Extern slobrok not ready.");
- }
- }
-
- public void merge(RoutingContext ctx) {
- DocumentProtocol.merge(ctx);
- }
-
- public void destroy() {
- }
-
- public MetricSet getMetrics() {
- return null;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.Hop; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingContext; + +import java.util.ArrayList; +import java.util.List; + +/** + * This policy implements the necessary logic to communicate with an external Vespa application and resolve its list of + * recipients using that other application's slobrok servers. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExternPolicy implements DocumentProtocolRoutingPolicy { + + private Supervisor orb = null; + private Mirror mirror = null; + private String pattern = null; + private String session = null; + private final String error; + private int offset = 0; + private int generation = 0; + private final List<Hop> recipients = new ArrayList<>(); + + /** + * Constructs a new instance of this policy. The argument given is the connection spec to the slobrok to use for + * resolving recipients, as well as the pattern to use when querying. This constructor does _not_ wait for the + * mirror to become ready. + * + * @param arg The slobrok connection spec. + */ + public ExternPolicy(String arg) { + if (arg == null || arg.length() == 0) { + error = "Expected parameter, got empty string."; + return; + } + String[] args = arg.split(";", 2); + if (args.length != 2 || args[0].length() == 0 || args[1].length() == 0) { + error = "Expected parameter on the form '<spec>;<pattern>', got '" + arg + "'."; + return; + } + int pos = args[1].lastIndexOf('/'); + if (pos < 0) { + error = "Expected pattern on the form '<service>/<session>', got '" + args[1] + "'."; + return; + } + SlobrokList slobroks = new SlobrokList(); + slobroks.setup(args[0].split(",")); + pattern = args[1]; + session = pattern.substring(pos); + orb = new Supervisor(new Transport()); + mirror = new Mirror(orb, slobroks); + error = null; + } + + /** + * This is a safety mechanism to allow the constructor to fail and signal that it can not be used. + * + * @return The error string, or null if no error. + */ + public String getError() { + return error; + } + + /** + * Returns the slobrok mirror used by this policy to resolve external recipients. + * + * @return The external mirror. + */ + public Mirror getMirror() { + return mirror; + } + + /** + * Returns the appropriate recipient hop. This method provides synchronized access to the internal mirror. + * + * @return The recipient hop to use. + */ + private synchronized Hop getRecipient() { + update(); + if (recipients.isEmpty()) { + return null; + } + int offset = ++this.offset & Integer.MAX_VALUE; // mask signed bit because of modulo + return new Hop(recipients.get(offset % recipients.size())); + } + + /** + * Updates the list of matching recipients by querying the extern slobrok. + */ + private void update() { + int upd = mirror.updates(); + if (generation != upd) { + generation = upd; + recipients.clear(); + Mirror.Entry[] arr = mirror.lookup(pattern); + for (Mirror.Entry entry : arr) { + recipients.add(Hop.parse(entry.getSpec() + session)); + } + } + } + + @Override + public void finalize() throws Throwable { + super.finalize(); + mirror.shutdown(); + orb.transport().shutdown().join(); + } + + public void select(RoutingContext ctx) { + if (error != null) { + ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, error); + } else if (mirror.ready()) { + Hop hop = getRecipient(); + if (hop != null) { + Route route = new Route(ctx.getRoute()); + route.setHop(0, hop); + ctx.addChild(route); + } else { + ctx.setError(ErrorCode.NO_ADDRESS_FOR_SERVICE, + "Could not resolve any recipients from '" + pattern + "'."); + } + } else { + ctx.setError(ErrorCode.APP_TRANSIENT_ERROR, "Extern slobrok not ready."); + } + } + + public void merge(RoutingContext ctx) { + DocumentProtocol.merge(ctx); + } + + public void destroy() { + } + + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListMessage.java index d38aa2e94f2..e7d623d86e2 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListMessage.java @@ -1,50 +1,50 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.BucketId;
-
-public class GetBucketListMessage extends DocumentMessage {
-
- private BucketId bucketId;
-
- GetBucketListMessage() {
- // must be deserialized into
- }
-
- public GetBucketListMessage(BucketId bucketId) {
- this.bucketId = bucketId;
- }
-
- public BucketId getBucketId() {
- return bucketId;
- }
-
- void setBucketId(BucketId id) {
- bucketId = id;
- }
-
- @Override
- public DocumentReply createReply() {
- return new StatBucketReply();
- }
-
- @Override
- public boolean hasSequenceId() {
- return true;
- }
-
- @Override
- public long getSequenceId() {
- return bucketId.getRawId();
- }
-
- @Override
- public int getApproxSize() {
- return super.getApproxSize() + 8;
- }
-
- @Override
- public int getType() {
- return DocumentProtocol.MESSAGE_GETBUCKETLIST;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; + +public class GetBucketListMessage extends DocumentMessage { + + private BucketId bucketId; + + GetBucketListMessage() { + // must be deserialized into + } + + public GetBucketListMessage(BucketId bucketId) { + this.bucketId = bucketId; + } + + public BucketId getBucketId() { + return bucketId; + } + + void setBucketId(BucketId id) { + bucketId = id; + } + + @Override + public DocumentReply createReply() { + return new StatBucketReply(); + } + + @Override + public boolean hasSequenceId() { + return true; + } + + @Override + public long getSequenceId() { + return bucketId.getRawId(); + } + + @Override + public int getApproxSize() { + return super.getApproxSize() + 8; + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_GETBUCKETLIST; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListReply.java index 07013507d91..51a5289c46f 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListReply.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetBucketListReply.java @@ -1,70 +1,70 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.BucketId;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class GetBucketListReply extends DocumentReply {
-
- public static class BucketInfo {
- BucketId bucket;
- String bucketInformation;
-
- BucketInfo() {
- // must be deserialized into
- }
-
- public BucketInfo(BucketId bucket, String bucketInformation) {
- this.bucket = bucket;
- this.bucketInformation = bucketInformation;
- }
-
- public BucketId getBucketId() {
- return bucket;
- }
-
- public String getBucketInformation() {
- return bucketInformation;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof BucketInfo)) {
- return false;
- }
- BucketInfo rhs = (BucketInfo)obj;
- if (bucket == null) {
- if (rhs.bucket != null) {
- return false;
- }
- } else if (!bucket.equals(rhs.bucket)) {
- return false;
- }
- if (bucketInformation == null) {
- if (rhs.bucketInformation != null) {
- return false;
- }
- } else if (!bucketInformation.equals(rhs.bucketInformation)) {
- return false;
- }
- return true;
- }
-
- @Override
- public String toString() {
- return String.format("BucketInfo(%s: %s)", bucket, bucketInformation);
- }
- }
-
- private final List<BucketInfo> buckets = new ArrayList<BucketInfo>();
-
- public GetBucketListReply() {
- super(DocumentProtocol.REPLY_GETBUCKETLIST);
- }
-
- public List<BucketInfo> getBuckets() {
- return buckets;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; + +import java.util.ArrayList; +import java.util.List; + +public class GetBucketListReply extends DocumentReply { + + public static class BucketInfo { + BucketId bucket; + String bucketInformation; + + BucketInfo() { + // must be deserialized into + } + + public BucketInfo(BucketId bucket, String bucketInformation) { + this.bucket = bucket; + this.bucketInformation = bucketInformation; + } + + public BucketId getBucketId() { + return bucket; + } + + public String getBucketInformation() { + return bucketInformation; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof BucketInfo)) { + return false; + } + BucketInfo rhs = (BucketInfo)obj; + if (bucket == null) { + if (rhs.bucket != null) { + return false; + } + } else if (!bucket.equals(rhs.bucket)) { + return false; + } + if (bucketInformation == null) { + if (rhs.bucketInformation != null) { + return false; + } + } else if (!bucketInformation.equals(rhs.bucketInformation)) { + return false; + } + return true; + } + + @Override + public String toString() { + return String.format("BucketInfo(%s: %s)", bucket, bucketInformation); + } + } + + private final List<BucketInfo> buckets = new ArrayList<BucketInfo>(); + + public GetBucketListReply() { + super(DocumentProtocol.REPLY_GETBUCKETLIST); + } + + public List<BucketInfo> getBuckets() { + return buckets; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java index cf66704d21f..f0a94fa4f7b 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java @@ -1,94 +1,94 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.DocumentId;
-
-import java.util.Arrays;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class GetDocumentMessage extends DocumentMessage {
-
- final static String DEFAULT_FIELD_SET = "[all]";
- private DocumentId documentId = null;
- private String fieldSet = DEFAULT_FIELD_SET;
-
- /**
- * Constructs a new message for deserialization.
- */
- GetDocumentMessage() {
- // empty
- }
-
- /**
- * Constructs a new document get message.
- *
- * @param documentId The identifier of the document to get.
- */
- public GetDocumentMessage(DocumentId documentId) {
- setDocumentId(documentId);
- }
-
- /**
- * Constructs a new document get message.
- *
- * @param documentId The identifier of the document to get.
- * @param fieldSet Which fields to retrieve from the document
- */
- public GetDocumentMessage(DocumentId documentId, String fieldSet) {
- setDocumentId(documentId);
- this.fieldSet = fieldSet;
- }
-
- /**
- * Returns the identifier of the document to retrieve.
- *
- * @return The document id.
- */
- public DocumentId getDocumentId() {
- return documentId;
- }
-
- /**
- * Sets the identifier of the document to retrieve.
- *
- * @param documentId The document id to set.
- */
- public void setDocumentId(DocumentId documentId) {
- if (documentId == null) {
- throw new IllegalArgumentException("Document id can not be null.");
- }
- this.documentId = documentId;
- }
-
- public String getFieldSet() {
- return fieldSet;
- }
-
- @Override
- public DocumentReply createReply() {
- return new GetDocumentReply();
- }
-
- @Override
- public int getApproxSize() {
- return super.getApproxSize() + 4 + documentId.toString().length();
- }
-
- @Override
- public boolean hasSequenceId() {
- return true;
- }
-
- @Override
- public long getSequenceId() {
- return Arrays.hashCode(documentId.getGlobalId());
- }
-
- @Override
- public int getType() {
- return DocumentProtocol.MESSAGE_GETDOCUMENT;
- }
-
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.DocumentId; + +import java.util.Arrays; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GetDocumentMessage extends DocumentMessage { + + final static String DEFAULT_FIELD_SET = "[all]"; + private DocumentId documentId = null; + private String fieldSet = DEFAULT_FIELD_SET; + + /** + * Constructs a new message for deserialization. + */ + GetDocumentMessage() { + // empty + } + + /** + * Constructs a new document get message. + * + * @param documentId The identifier of the document to get. + */ + public GetDocumentMessage(DocumentId documentId) { + setDocumentId(documentId); + } + + /** + * Constructs a new document get message. + * + * @param documentId The identifier of the document to get. + * @param fieldSet Which fields to retrieve from the document + */ + public GetDocumentMessage(DocumentId documentId, String fieldSet) { + setDocumentId(documentId); + this.fieldSet = fieldSet; + } + + /** + * Returns the identifier of the document to retrieve. + * + * @return The document id. + */ + public DocumentId getDocumentId() { + return documentId; + } + + /** + * Sets the identifier of the document to retrieve. + * + * @param documentId The document id to set. + */ + public void setDocumentId(DocumentId documentId) { + if (documentId == null) { + throw new IllegalArgumentException("Document id can not be null."); + } + this.documentId = documentId; + } + + public String getFieldSet() { + return fieldSet; + } + + @Override + public DocumentReply createReply() { + return new GetDocumentReply(); + } + + @Override + public int getApproxSize() { + return super.getApproxSize() + 4 + documentId.toString().length(); + } + + @Override + public boolean hasSequenceId() { + return true; + } + + @Override + public long getSequenceId() { + return Arrays.hashCode(documentId.getGlobalId()); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_GETDOCUMENT; + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java index f5f687bb18b..0d1413c3bfe 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java @@ -1,108 +1,108 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.Document;
-import com.yahoo.document.serialization.DocumentDeserializer;
-
-import java.nio.ByteBuffer;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class GetDocumentReply extends DocumentAcceptedReply {
-
- private DocumentDeserializer buffer = null;
- private Document document = null;
- private long lastModified = 0;
- private LazyDecoder decoder = null;
-
- /**
- * Constructs a new reply for deserialization.
- */
- GetDocumentReply() {
- super(DocumentProtocol.REPLY_GETDOCUMENT);
- }
-
- /**
- * Constructs a new reply to lazily deserialize from a byte buffer.
- * @param decoder The decoder to use for deserialization.
- * @param buf A byte buffer that contains a serialized reply.
- */
- GetDocumentReply(LazyDecoder decoder, DocumentDeserializer buf) {
- super(DocumentProtocol.REPLY_GETDOCUMENT);
- this.decoder = decoder;
- buffer = buf;
- }
-
- /**
- * Constructs a new document get reply.
- *
- * @param doc The document requested.
- */
- public GetDocumentReply(Document doc) {
- super(DocumentProtocol.REPLY_GETDOCUMENT);
- document = doc;
- }
-
- /**
- * This method will make sure that any serialized content is deserialized into proper message content on first
- * entry. Any subsequent entry into this function will do nothing.
- */
- private void deserialize() {
- if (decoder != null && buffer != null) {
- decoder.decode(this, buffer);
- decoder = null;
- buffer = null;
- }
- }
-
- /**
- * Returns the document retrieved.
- *
- * @return The document.
- */
- public Document getDocument() {
- deserialize();
- return document;
- }
-
- /**
- * Sets the document of this reply.
- *
- * @param doc The document to set.
- */
- public void setDocument(Document doc) {
- buffer = null;
- decoder = null;
- document = doc;
- lastModified = document != null && document.getLastModified() != null ? document.getLastModified() : 0;
- }
-
- /**
- * Returns the date the document was last modified.
- *
- * @return The date.
- */
- public long getLastModified() {
- deserialize();
- return lastModified;
- }
-
- /**
- * Set the date the document was last modified.
- *
- * @param modified The date.
- */
- void setLastModified(long modified) {
- lastModified = modified;
- }
-
- /**
- * Returns the internal buffer to deserialize from, may be null.
- *
- * @return The buffer.
- */
- public ByteBuffer getSerializedBuffer() {
- return buffer != null ? buffer.getBuf().getByteBuffer() : null;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.Document; +import com.yahoo.document.serialization.DocumentDeserializer; + +import java.nio.ByteBuffer; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GetDocumentReply extends DocumentAcceptedReply { + + private DocumentDeserializer buffer = null; + private Document document = null; + private long lastModified = 0; + private LazyDecoder decoder = null; + + /** + * Constructs a new reply for deserialization. + */ + GetDocumentReply() { + super(DocumentProtocol.REPLY_GETDOCUMENT); + } + + /** + * Constructs a new reply to lazily deserialize from a byte buffer. + * @param decoder The decoder to use for deserialization. + * @param buf A byte buffer that contains a serialized reply. + */ + GetDocumentReply(LazyDecoder decoder, DocumentDeserializer buf) { + super(DocumentProtocol.REPLY_GETDOCUMENT); + this.decoder = decoder; + buffer = buf; + } + + /** + * Constructs a new document get reply. + * + * @param doc The document requested. + */ + public GetDocumentReply(Document doc) { + super(DocumentProtocol.REPLY_GETDOCUMENT); + document = doc; + } + + /** + * This method will make sure that any serialized content is deserialized into proper message content on first + * entry. Any subsequent entry into this function will do nothing. + */ + private void deserialize() { + if (decoder != null && buffer != null) { + decoder.decode(this, buffer); + decoder = null; + buffer = null; + } + } + + /** + * Returns the document retrieved. + * + * @return The document. + */ + public Document getDocument() { + deserialize(); + return document; + } + + /** + * Sets the document of this reply. + * + * @param doc The document to set. + */ + public void setDocument(Document doc) { + buffer = null; + decoder = null; + document = doc; + lastModified = document != null && document.getLastModified() != null ? document.getLastModified() : 0; + } + + /** + * Returns the date the document was last modified. + * + * @return The date. + */ + public long getLastModified() { + deserialize(); + return lastModified; + } + + /** + * Set the date the document was last modified. + * + * @param modified The date. + */ + void setLastModified(long modified) { + lastModified = modified; + } + + /** + * Returns the internal buffer to deserialize from, may be null. + * + * @return The buffer. + */ + public ByteBuffer getSerializedBuffer() { + return buffer != null ? buffer.getBuf().getByteBuffer() : null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LazyDecoder.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LazyDecoder.java index 2ac7f716850..325b8ab44af 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LazyDecoder.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LazyDecoder.java @@ -8,4 +8,4 @@ public interface LazyDecoder { public void decode(Routable obj, DocumentDeserializer buf); -}
\ No newline at end of file +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java index 74ca65df547..c4f190559d3 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java @@ -1,138 +1,138 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.jrt.slobrok.api.Mirror;
-import com.yahoo.messagebus.metrics.MetricSet;
-import com.yahoo.messagebus.routing.*;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This policy implements the logic to prefer local services that matches a slobrok pattern.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class LocalServicePolicy implements DocumentProtocolRoutingPolicy {
-
- private final String localAddress;
- private Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>();
-
- /**
- * Constructs a policy that will choose local services that match the slobrok pattern in which this policy occured.
- * If no local service can be found, this policy simply returns the asterisk to allow the network to choose any.
- *
- * @param param The address to use for this, if empty this will resolve to hostname.
- */
- public LocalServicePolicy(String param) {
- localAddress = (param != null && param.length() > 0) ? param : null;
- }
-
- // Inherit doc from RoutingPolicy.
- public void select(RoutingContext ctx) {
- Route route = new Route(ctx.getRoute());
- route.setHop(0, getRecipient(ctx));
- ctx.addChild(route);
- }
-
- // Inherit doc from RoutingPolicy.
- public void merge(RoutingContext ctx) {
- DocumentProtocol.merge(ctx);
- }
-
- /**
- * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to
- * the internal cache.
- *
- * @param ctx The routing context.
- * @return The recipient hop to use.
- */
- private synchronized Hop getRecipient(RoutingContext ctx) {
- CacheEntry entry = update(ctx);
- if (entry.recipients.isEmpty()) {
- Hop hop = new Hop(ctx.getRoute().getHop(0));
- hop.setDirective(ctx.getDirectiveIndex(), new VerbatimDirective("*"));
- return hop;
- }
- if (++entry.offset >= entry.recipients.size()) {
- entry.offset = 0;
- }
- return new Hop(entry.recipients.get(entry.offset));
- }
-
- /**
- * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is
- * handled outside of it.
- *
- * @param ctx The routing context.
- * @return The updated cache entry.
- */
- private CacheEntry update(RoutingContext ctx) {
- String key = getCacheKey(ctx);
- CacheEntry entry = cache.get(key);
- if (entry == null) {
- entry = new CacheEntry();
- cache.put(key, entry);
- }
- int upd = ctx.getMirror().updates();
- if (entry.generation != upd) {
- entry.generation = upd;
- entry.recipients.clear();
-
- Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix());
- String self = localAddress != null ? localAddress : toAddress(ctx.getMessageBus().getConnectionSpec());
- for (Mirror.Entry item : arr) {
- if (self.equals(toAddress(item.getSpec()))) {
- entry.recipients.add(Hop.parse(item.getName()));
- }
- }
- }
- return entry;
- }
-
- /**
- * Returns a cache key for this instance of the policy. Because behaviour is based on the hop in which the policy
- * occurs, the cache key is the hop string itself.
- *
- * @param ctx The routing context.
- * @return The cache key.
- */
- private String getCacheKey(RoutingContext ctx) {
- return ctx.getRoute().getHop(0).toString();
- }
-
- /**
- * Defines the necessary cache data.
- */
- private class CacheEntry {
- private final List<Hop> recipients = new ArrayList<Hop>();
- private int generation = 0;
- private int offset = 0;
- }
-
- /**
- * Searches the given connection spec for a hostname or IP address. If an address is not found, this method returns
- * null.
- *
- * @param connection The connection spec to search.
- * @return The address, may be null.
- */
- private static String toAddress(String connection) {
- if (connection.startsWith("tcp/")) {
- int pos = connection.indexOf(':');
- if (pos > 4) {
- return connection.substring(4, pos);
- }
- }
- return null;
- }
-
- public void destroy() {
- }
-
- public MetricSet getMetrics() {
- return null;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This policy implements the logic to prefer local services that matches a slobrok pattern. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class LocalServicePolicy implements DocumentProtocolRoutingPolicy { + + private final String localAddress; + private Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>(); + + /** + * Constructs a policy that will choose local services that match the slobrok pattern in which this policy occured. + * If no local service can be found, this policy simply returns the asterisk to allow the network to choose any. + * + * @param param The address to use for this, if empty this will resolve to hostname. + */ + public LocalServicePolicy(String param) { + localAddress = (param != null && param.length() > 0) ? param : null; + } + + // Inherit doc from RoutingPolicy. + public void select(RoutingContext ctx) { + Route route = new Route(ctx.getRoute()); + route.setHop(0, getRecipient(ctx)); + ctx.addChild(route); + } + + // Inherit doc from RoutingPolicy. + public void merge(RoutingContext ctx) { + DocumentProtocol.merge(ctx); + } + + /** + * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to + * the internal cache. + * + * @param ctx The routing context. + * @return The recipient hop to use. + */ + private synchronized Hop getRecipient(RoutingContext ctx) { + CacheEntry entry = update(ctx); + if (entry.recipients.isEmpty()) { + Hop hop = new Hop(ctx.getRoute().getHop(0)); + hop.setDirective(ctx.getDirectiveIndex(), new VerbatimDirective("*")); + return hop; + } + if (++entry.offset >= entry.recipients.size()) { + entry.offset = 0; + } + return new Hop(entry.recipients.get(entry.offset)); + } + + /** + * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is + * handled outside of it. + * + * @param ctx The routing context. + * @return The updated cache entry. + */ + private CacheEntry update(RoutingContext ctx) { + String key = getCacheKey(ctx); + CacheEntry entry = cache.get(key); + if (entry == null) { + entry = new CacheEntry(); + cache.put(key, entry); + } + int upd = ctx.getMirror().updates(); + if (entry.generation != upd) { + entry.generation = upd; + entry.recipients.clear(); + + Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); + String self = localAddress != null ? localAddress : toAddress(ctx.getMessageBus().getConnectionSpec()); + for (Mirror.Entry item : arr) { + if (self.equals(toAddress(item.getSpec()))) { + entry.recipients.add(Hop.parse(item.getName())); + } + } + } + return entry; + } + + /** + * Returns a cache key for this instance of the policy. Because behaviour is based on the hop in which the policy + * occurs, the cache key is the hop string itself. + * + * @param ctx The routing context. + * @return The cache key. + */ + private String getCacheKey(RoutingContext ctx) { + return ctx.getRoute().getHop(0).toString(); + } + + /** + * Defines the necessary cache data. + */ + private class CacheEntry { + private final List<Hop> recipients = new ArrayList<Hop>(); + private int generation = 0; + private int offset = 0; + } + + /** + * Searches the given connection spec for a hostname or IP address. If an address is not found, this method returns + * null. + * + * @param connection The connection spec to search. + * @return The address, may be null. + */ + private static String toAddress(String connection) { + if (connection.startsWith("tcp/")) { + int pos = connection.indexOf(':'); + if (pos > 4) { + return connection.substring(4, pos); + } + } + return null; + } + + public void destroy() { + } + + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentMessage.java index f6fcb5965ad..e184de7d4a4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentMessage.java @@ -1,101 +1,101 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.DocumentRemove;
-import com.yahoo.document.TestAndSetCondition;
-
-import java.util.Arrays;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class RemoveDocumentMessage extends TestAndSetMessage {
- private DocumentRemove remove = null;
-
- /**
- * Constructs a new message for deserialization.
- */
- RemoveDocumentMessage() {
- // empty
- }
-
- /**
- * Constructs a new document remove message.
- *
- * @param documentId The identifier of the document to remove.
- */
- public RemoveDocumentMessage(DocumentId documentId) {
- remove = new DocumentRemove(documentId);
- }
-
- /**
- * Constructs a new document remove message.
- *
- * @param remove The DocumentRemove operation to perform
- */
- public RemoveDocumentMessage(DocumentRemove remove) {
- this.remove = remove;
- }
-
- /**
- * Returns the identifier of the document to remove.
- *
- * @return The document id.
- */
- public DocumentId getDocumentId() {
- return remove.getId();
- }
-
- /**
- * Sets the identifier of the document to remove.
- *
- * @param documentId The document id to set.
- */
- public void setDocumentId(DocumentId documentId) {
- if (documentId == null) {
- throw new IllegalArgumentException("Document id can not be null.");
- }
-
- remove = new DocumentRemove(documentId);
- }
-
- @Override
- public DocumentReply createReply() {
- return new RemoveDocumentReply();
- }
-
- @Override
- public int getApproxSize() {
- return super.getApproxSize() + 4 + remove.getId().toString().length();
- }
-
- @Override
- public boolean hasSequenceId() {
- return true;
- }
-
- @Override
- public long getSequenceId() {
- return Arrays.hashCode(remove.getId().getGlobalId());
- }
-
- @Override
- public int getType() {
- return DocumentProtocol.MESSAGE_REMOVEDOCUMENT;
- }
-
- @Override
- public void setCondition(TestAndSetCondition condition) {
- remove.setCondition(condition);
- }
-
- @Override
- public TestAndSetCondition getCondition() {
- return remove.getCondition();
- }
-
- public DocumentRemove getDocumentRemove() {
- return remove;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.TestAndSetCondition; + +import java.util.Arrays; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RemoveDocumentMessage extends TestAndSetMessage { + private DocumentRemove remove = null; + + /** + * Constructs a new message for deserialization. + */ + RemoveDocumentMessage() { + // empty + } + + /** + * Constructs a new document remove message. + * + * @param documentId The identifier of the document to remove. + */ + public RemoveDocumentMessage(DocumentId documentId) { + remove = new DocumentRemove(documentId); + } + + /** + * Constructs a new document remove message. + * + * @param remove The DocumentRemove operation to perform + */ + public RemoveDocumentMessage(DocumentRemove remove) { + this.remove = remove; + } + + /** + * Returns the identifier of the document to remove. + * + * @return The document id. + */ + public DocumentId getDocumentId() { + return remove.getId(); + } + + /** + * Sets the identifier of the document to remove. + * + * @param documentId The document id to set. + */ + public void setDocumentId(DocumentId documentId) { + if (documentId == null) { + throw new IllegalArgumentException("Document id can not be null."); + } + + remove = new DocumentRemove(documentId); + } + + @Override + public DocumentReply createReply() { + return new RemoveDocumentReply(); + } + + @Override + public int getApproxSize() { + return super.getApproxSize() + 4 + remove.getId().toString().length(); + } + + @Override + public boolean hasSequenceId() { + return true; + } + + @Override + public long getSequenceId() { + return Arrays.hashCode(remove.getId().getGlobalId()); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_REMOVEDOCUMENT; + } + + @Override + public void setCondition(TestAndSetCondition condition) { + remove.setCondition(condition); + } + + @Override + public TestAndSetCondition getCondition() { + return remove.getCondition(); + } + + public DocumentRemove getDocumentRemove() { + return remove; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentReply.java index c259aaa5731..c60bf65fed4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentReply.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RemoveDocumentReply.java @@ -1,35 +1,35 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class RemoveDocumentReply extends WriteDocumentReply {
-
- private boolean found = true;
-
- /**
- * Constructs a new reply with no content.
- */
- public RemoveDocumentReply() {
- super(DocumentProtocol.REPLY_REMOVEDOCUMENT);
- }
-
- /**
- * Returns whether or not the document was found and removed.
- *
- * @return True if document was found.
- */
- public boolean wasFound() {
- return found;
- }
-
- /**
- * Set whether or not the document was found and removed.
- *
- * @param found True if the document was found.
- */
- public void setWasFound(boolean found) {
- this.found = found;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RemoveDocumentReply extends WriteDocumentReply { + + private boolean found = true; + + /** + * Constructs a new reply with no content. + */ + public RemoveDocumentReply() { + super(DocumentProtocol.REPLY_REMOVEDOCUMENT); + } + + /** + * Returns whether or not the document was found and removed. + * + * @return True if document was found. + */ + public boolean wasFound() { + return found; + } + + /** + * Set whether or not the document was found and removed. + * + * @param found True if the document was found. + */ + public void setWasFound(boolean found) { + this.found = found; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ReplyMerger.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ReplyMerger.java index e0873a840d0..97ecc20dd3d 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ReplyMerger.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ReplyMerger.java @@ -1,111 +1,111 @@ -// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.collections.Tuple2;
-import com.yahoo.messagebus.EmptyReply;
-import com.yahoo.messagebus.Reply;
-
-/**
- * Encapsulated logic for merging replies from 1-n related DocumentProtocol messages.
- * For internal use only. Not multithread safe.
- */
-final class ReplyMerger {
-
- private Reply successReply = null;
- private int successIndex = -1;
- private Reply error = null;
- private Reply ignore = null;
-
- public void merge(int i, Reply r) {
- if (r.hasErrors()) {
- mergeAllReplyErrors(r);
- } else {
- updateStateWithSuccessfulReply(i, r);
- }
- }
-
- private boolean resourceWasFound(Reply r) {
- if (r instanceof RemoveDocumentReply) {
- return ((RemoveDocumentReply) r).wasFound();
- }
- if (r instanceof UpdateDocumentReply) {
- return ((UpdateDocumentReply) r).wasFound();
- }
- if (r instanceof GetDocumentReply) {
- return ((GetDocumentReply) r).getLastModified() > 0;
- }
- return false;
- }
-
- private boolean replyIsBetterThanCurrent(Reply r) {
- return resourceWasFound(r) && !resourceWasFound(successReply);
- }
-
- private void updateStateWithSuccessfulReply(int i, Reply r) {
- if (successReply == null || replyIsBetterThanCurrent(r)) {
- setCurrentBestReply(i, r);
- }
- }
-
- private void setCurrentBestReply(int i, Reply r) {
- successReply = r;
- successIndex = i;
- }
-
- private void mergeAllReplyErrors(Reply r) {
- if (handleReplyWithOnlyIgnoredErrors(r)) {
- return;
- }
- if (error == null) {
- error = new EmptyReply();
- r.swapState(error);
- return;
- }
- for (int j = 0; j < r.getNumErrors(); ++j) {
- error.addError(r.getError(j));
- }
- }
-
- private boolean handleReplyWithOnlyIgnoredErrors(Reply r) {
- if (DocumentProtocol.hasOnlyErrorsOfType(r, DocumentProtocol.ERROR_MESSAGE_IGNORED)) {
- if (ignore == null) {
- ignore = new EmptyReply();
- }
- ignore.addError(r.getError(0));
- return true;
- }
- return false;
- }
-
- private boolean shouldReturnErrorReply() {
- return (error != null || (ignore != null && successReply == null));
- }
-
- private Tuple2<Integer, Reply> createMergedErrorReplyResult() {
- if (error != null) {
- return new Tuple2<>(null, error);
- }
- if (ignore != null && successReply == null) {
- return new Tuple2<>(null, ignore);
- }
- throw new IllegalStateException("createMergedErrorReplyResult called without error");
- }
-
- private boolean successfullyMergedAtLeastOneReply() {
- return successReply != null;
- }
-
- private Tuple2<Integer, Reply> createEmptyReplyResult() {
- return new Tuple2<>(null, (Reply)new EmptyReply());
- }
-
- public Tuple2<Integer, Reply> mergedReply() {
- if (shouldReturnErrorReply()) {
- return createMergedErrorReplyResult();
- } else if (!successfullyMergedAtLeastOneReply()) {
- return createEmptyReplyResult();
- }
- return new Tuple2<>(successIndex, successReply);
- }
-
-}
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.collections.Tuple2; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.Reply; + +/** + * Encapsulated logic for merging replies from 1-n related DocumentProtocol messages. + * For internal use only. Not multithread safe. + */ +final class ReplyMerger { + + private Reply successReply = null; + private int successIndex = -1; + private Reply error = null; + private Reply ignore = null; + + public void merge(int i, Reply r) { + if (r.hasErrors()) { + mergeAllReplyErrors(r); + } else { + updateStateWithSuccessfulReply(i, r); + } + } + + private boolean resourceWasFound(Reply r) { + if (r instanceof RemoveDocumentReply) { + return ((RemoveDocumentReply) r).wasFound(); + } + if (r instanceof UpdateDocumentReply) { + return ((UpdateDocumentReply) r).wasFound(); + } + if (r instanceof GetDocumentReply) { + return ((GetDocumentReply) r).getLastModified() > 0; + } + return false; + } + + private boolean replyIsBetterThanCurrent(Reply r) { + return resourceWasFound(r) && !resourceWasFound(successReply); + } + + private void updateStateWithSuccessfulReply(int i, Reply r) { + if (successReply == null || replyIsBetterThanCurrent(r)) { + setCurrentBestReply(i, r); + } + } + + private void setCurrentBestReply(int i, Reply r) { + successReply = r; + successIndex = i; + } + + private void mergeAllReplyErrors(Reply r) { + if (handleReplyWithOnlyIgnoredErrors(r)) { + return; + } + if (error == null) { + error = new EmptyReply(); + r.swapState(error); + return; + } + for (int j = 0; j < r.getNumErrors(); ++j) { + error.addError(r.getError(j)); + } + } + + private boolean handleReplyWithOnlyIgnoredErrors(Reply r) { + if (DocumentProtocol.hasOnlyErrorsOfType(r, DocumentProtocol.ERROR_MESSAGE_IGNORED)) { + if (ignore == null) { + ignore = new EmptyReply(); + } + ignore.addError(r.getError(0)); + return true; + } + return false; + } + + private boolean shouldReturnErrorReply() { + return (error != null || (ignore != null && successReply == null)); + } + + private Tuple2<Integer, Reply> createMergedErrorReplyResult() { + if (error != null) { + return new Tuple2<>(null, error); + } + if (ignore != null && successReply == null) { + return new Tuple2<>(null, ignore); + } + throw new IllegalStateException("createMergedErrorReplyResult called without error"); + } + + private boolean successfullyMergedAtLeastOneReply() { + return successReply != null; + } + + private Tuple2<Integer, Reply> createEmptyReplyResult() { + return new Tuple2<>(null, (Reply)new EmptyReply()); + } + + public Tuple2<Integer, Reply> mergedReply() { + if (shouldReturnErrorReply()) { + return createMergedErrorReplyResult(); + } else if (!successfullyMergedAtLeastOneReply()) { + return createEmptyReplyResult(); + } + return new Tuple2<>(successIndex, successReply); + } + +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java index f0e49146851..bbacefdc80b 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java @@ -1,125 +1,125 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.jrt.slobrok.api.Mirror;
-import com.yahoo.messagebus.EmptyReply;
-import com.yahoo.messagebus.Error;
-import com.yahoo.messagebus.ErrorCode;
-import com.yahoo.messagebus.Reply;
-import com.yahoo.messagebus.metrics.MetricSet;
-import com.yahoo.messagebus.routing.Hop;
-import com.yahoo.messagebus.routing.Route;
-import com.yahoo.messagebus.routing.RoutingContext;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This policy implements round-robin selection of the configured recipients that are currently registered in slobrok.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class RoundRobinPolicy implements DocumentProtocolRoutingPolicy {
-
- private final Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>();
-
- // Inherit doc from RoutingPolicy.
- public void select(RoutingContext ctx) {
- Hop hop = getRecipient(ctx);
- if (hop != null) {
- Route route = new Route(ctx.getRoute());
- route.setHop(0, hop);
- ctx.addChild(route);
- } else {
- Reply reply = new EmptyReply();
- reply.addError(new Error(ErrorCode.NO_ADDRESS_FOR_SERVICE,
- "None of the configured recipients are currently available."));
- ctx.setReply(reply);
- }
- }
-
- // Inherit doc from RoutingPolicy.
- public void merge(RoutingContext ctx) {
- DocumentProtocol.merge(ctx);
- }
-
- /**
- * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to
- * the internal cache.
- *
- * @param ctx The routing context.
- * @return The recipient hop to use.
- */
- private synchronized Hop getRecipient(RoutingContext ctx) {
- CacheEntry entry = update(ctx);
- if (entry.recipients.isEmpty()) {
- return null;
- }
- if (++entry.offset >= entry.recipients.size()) {
- entry.offset = 0;
- }
- return new Hop(entry.recipients.get(entry.offset));
- }
-
- /**
- * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is
- * handled outside of it.
- *
- * @param ctx The routing context.
- * @return The updated cache entry.
- */
- private CacheEntry update(RoutingContext ctx) {
- String key = getCacheKey(ctx);
- CacheEntry entry = cache.get(key);
- if (entry == null) {
- entry = new CacheEntry();
- cache.put(key, entry);
- }
-
- int upd = ctx.getMirror().updates();
- if (entry.generation != upd) {
- entry.generation = upd;
- entry.recipients.clear();
- for (int i = 0; i < ctx.getNumRecipients(); ++i) {
- Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getRecipient(i).getHop(0).toString());
- for (Mirror.Entry item : arr) {
- entry.recipients.add(Hop.parse(item.getName()));
- }
- }
- }
- return entry;
- }
-
- /**
- * Returns a cache key for this instance of the policy. Because behaviour is based on the recipient list of this
- * policy, the cache key is the concatenated string of recipient routes.
- *
- * @param ctx The routing context.
- * @return The cache key.
- */
- private String getCacheKey(RoutingContext ctx) {
- StringBuilder ret = new StringBuilder();
- for (int i = 0; i < ctx.getNumRecipients(); ++i) {
- ret.append(ctx.getRecipient(i).getHop(0).toString()).append(" ");
- }
- return ret.toString();
- }
-
- /**
- * Defines the necessary cache data.
- */
- private class CacheEntry {
- private final List<Hop> recipients = new ArrayList<Hop>();
- private int generation = 0;
- private int offset = 0;
- }
-
- public void destroy() {
- }
-
- public MetricSet getMetrics() {
- return null;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.Hop; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingContext; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This policy implements round-robin selection of the configured recipients that are currently registered in slobrok. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RoundRobinPolicy implements DocumentProtocolRoutingPolicy { + + private final Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>(); + + // Inherit doc from RoutingPolicy. + public void select(RoutingContext ctx) { + Hop hop = getRecipient(ctx); + if (hop != null) { + Route route = new Route(ctx.getRoute()); + route.setHop(0, hop); + ctx.addChild(route); + } else { + Reply reply = new EmptyReply(); + reply.addError(new Error(ErrorCode.NO_ADDRESS_FOR_SERVICE, + "None of the configured recipients are currently available.")); + ctx.setReply(reply); + } + } + + // Inherit doc from RoutingPolicy. + public void merge(RoutingContext ctx) { + DocumentProtocol.merge(ctx); + } + + /** + * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to + * the internal cache. + * + * @param ctx The routing context. + * @return The recipient hop to use. + */ + private synchronized Hop getRecipient(RoutingContext ctx) { + CacheEntry entry = update(ctx); + if (entry.recipients.isEmpty()) { + return null; + } + if (++entry.offset >= entry.recipients.size()) { + entry.offset = 0; + } + return new Hop(entry.recipients.get(entry.offset)); + } + + /** + * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is + * handled outside of it. + * + * @param ctx The routing context. + * @return The updated cache entry. + */ + private CacheEntry update(RoutingContext ctx) { + String key = getCacheKey(ctx); + CacheEntry entry = cache.get(key); + if (entry == null) { + entry = new CacheEntry(); + cache.put(key, entry); + } + + int upd = ctx.getMirror().updates(); + if (entry.generation != upd) { + entry.generation = upd; + entry.recipients.clear(); + for (int i = 0; i < ctx.getNumRecipients(); ++i) { + Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getRecipient(i).getHop(0).toString()); + for (Mirror.Entry item : arr) { + entry.recipients.add(Hop.parse(item.getName())); + } + } + } + return entry; + } + + /** + * Returns a cache key for this instance of the policy. Because behaviour is based on the recipient list of this + * policy, the cache key is the concatenated string of recipient routes. + * + * @param ctx The routing context. + * @return The cache key. + */ + private String getCacheKey(RoutingContext ctx) { + StringBuilder ret = new StringBuilder(); + for (int i = 0; i < ctx.getNumRecipients(); ++i) { + ret.append(ctx.getRecipient(i).getHop(0).toString()).append(" "); + } + return ret.toString(); + } + + /** + * Defines the necessary cache data. + */ + private class CacheEntry { + private final List<Hop> recipients = new ArrayList<Hop>(); + private int generation = 0; + private int offset = 0; + } + + public void destroy() { + } + + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java index 6f044a1951f..5f29f3600e4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java @@ -1,237 +1,237 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.component.Version;
-import com.yahoo.component.VersionSpecification;
-import com.yahoo.concurrent.CopyOnWriteHashMap;
-import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.document.serialization.*;
-import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
-import com.yahoo.io.GrowableByteBuffer;
-import com.yahoo.log.LogLevel;
-import com.yahoo.messagebus.Routable;
-
-import java.util.*;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * This class encapsulates the logic required to map routable type and version to a corresponding {@link
- * RoutableFactory}. It is owned and accessed through a {@link DocumentProtocol} instance. This class uses a factory
- * cache to reduce the latency of matching version specifications to actual versions when resolving factories.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-final class RoutableRepository {
-
- private static final Logger log = Logger.getLogger(RoutableRepository.class.getName());
- private final CopyOnWriteHashMap<Integer, VersionMap> factoryTypes = new CopyOnWriteHashMap<>();
- private final CopyOnWriteHashMap<CacheKey, RoutableFactory> cache = new CopyOnWriteHashMap<>();
- private LoadTypeSet loadTypes;
-
- public RoutableRepository(LoadTypeSet set) {
- loadTypes = set;
- }
-
- /**
- * Decodes a {@link Routable} from the given byte array. This uses the content of the byte array to dispatch the
- * decode request to the appropriate {@link RoutableFactory} that was previously registered.
- *
- * If a routable can not be decoded, this method returns null.
- *
- * @param version The version of the encoded routable.
- * @param data The byte array containing the encoded routable.
- * @return The decoded routable.
- */
- Routable decode(DocumentTypeManager docMan, Version version, byte[] data) {
- if (data == null || data.length == 0) {
- log.log(LogLevel.ERROR, "Received empty byte array for deserialization.");
- return null;
- }
- DocumentDeserializer in;
-
- if (version.getMajor() >= 5) {
- in = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data));
- } else {
- in = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(data));
- }
-
- int type = in.getInt(null);
- RoutableFactory factory = getFactory(version, type);
- if (factory == null) {
- log.log(LogLevel.ERROR, "No routable factory found for routable type " + type +
- " (version " + version + ").");
- return null;
- }
- Routable ret = factory.decode(in, loadTypes);
- if (ret == null) {
- log.log(LogLevel.ERROR, "Routable factory " + factory.getClass().getName() + " failed to deserialize " +
- "routable of type " + type + " (version " + version + ").");
- log.log(LogLevel.ERROR, Arrays.toString(data));
- return null;
- }
- return ret;
- }
-
- /**
- * Encodes a {@link Routable} into a byte array. This dispatches the encode request to the appropriate {@link
- * RoutableFactory} that was previously registered.
- *
- * If a routable can not be encoded, this method returns an empty byte array.
- *
- * @param version The version to encode the routable as.
- * @param obj The routable to encode.
- * @return The byte array containing the encoded routable.
- */
- byte[] encode(Version version, Routable obj) {
- int type = obj.getType();
- RoutableFactory factory = getFactory(version, type);
- if (factory == null) {
- log.log(LogLevel.ERROR, "No routable factory found for routable type " + type +
- " (version " + version + ").");
- return new byte[0];
- }
- DocumentSerializer out;
-
- if (version.getMajor() >= 5) {
- out = DocumentSerializerFactory.createHead(new GrowableByteBuffer(8192));
- } else {
- out = DocumentSerializerFactory.create42(new GrowableByteBuffer(8192));
- }
-
- out.putInt(null, type);
- if (!factory.encode(obj, out)) {
- log.log(LogLevel.ERROR, "Routable factory " + factory.getClass().getName() + " failed to serialize " +
- "routable of type " + type + " (version " + version + ").");
- return new byte[0];
- }
- byte[] ret = new byte[out.getBuf().position()];
- out.getBuf().rewind();
- out.getBuf().get(ret);
- return ret;
- }
-
- /**
- * Registers a routable factory for a given version and routable type.
- *
- * @param version The version specification that the given factory supports.
- * @param type The routable type that the given factory supports.
- * @param factory The routable factory to register.
- */
- void putFactory(VersionSpecification version, int type, RoutableFactory factory) {
- VersionMap versionMap = factoryTypes.get(type);
- if (versionMap == null) {
- versionMap = new VersionMap();
-
- factoryTypes.put(type, versionMap);
- }
- if (versionMap.putFactory(version, factory)) {
- cache.clear();
- }
- }
-
- /**
- * Returns the routable factory for a given version and routable type.
- *
- * @param version The version that the factory must support.
- * @param type The routable type that the factory must support.
- * @return The routable factory matching the criteria, or null.
- */
- RoutableFactory getFactory(Version version, int type) {
- CacheKey cacheKey = new CacheKey(version, type);
- RoutableFactory factory = cache.get(cacheKey);
- if (factory != null) {
- return factory;
- }
- VersionMap versionMap = factoryTypes.get(type);
- if (versionMap == null) {
- return null;
- }
- factory = versionMap.getFactory(version);
- if (factory == null) {
- return null;
- }
- cache.put(cacheKey, factory);
- return factory;
- }
-
- /**
- * Returns a list of routable types that support the given version.
- *
- * @param version The version to return types for.
- * @return The list of supported types.
- */
- List<Integer> getRoutableTypes(Version version) {
- List<Integer> ret = new ArrayList<>();
- for (Map.Entry<Integer, VersionMap> entry : factoryTypes.entrySet()) {
- if (entry.getValue().getFactory(version) != null) {
- ret.add(entry.getKey());
- }
- }
- return ret;
- }
-
- /**
- * Internal helper class that implements a map from {@link VersionSpecification} to {@link RoutableFactory}.
- */
- private static class VersionMap {
-
- final Map<VersionSpecification, RoutableFactory> factoryVersions = new HashMap<>();
-
- boolean putFactory(VersionSpecification version, RoutableFactory factory) {
- return factoryVersions.put(version, factory) == null;
- }
-
- RoutableFactory getFactory(Version version) {
- VersionSpecification versionSpec = version.toSpecification();
-
- // Retrieve the factory with the highest version lower than or equal to actual version
- return factoryVersions.entrySet().stream()
- // Drop factories that have a higher version than actual version
- .filter(entry -> entry.getKey().compareTo(versionSpec) <= 0)
-
- // Get the factory with the highest version
- .max((entry1, entry2) -> entry1.getKey().compareTo(entry2.getKey()))
- .map(Map.Entry::getValue)
-
- // Return factory or null if no suitable factory found
- .orElse(null);
- }
- }
-
- /**
- * Internal helper class that implements a cache key for mapping a {@link Version} and routable type to a {@link
- * RoutableFactory}.
- */
- private static class CacheKey {
-
- final Version version;
- final int type;
-
- public CacheKey(Version version, int type) {
- this.version = version;
- this.type = type;
- }
-
- @Override
- public int hashCode() {
- return version.hashCode() + type;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof CacheKey)) {
- return false;
- }
- CacheKey rhs = (CacheKey)obj;
- if (!version.equals(rhs.version)) {
- return false;
- }
- if (type != rhs.type) {
- return false;
- }
- return true;
- }
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.component.Version; +import com.yahoo.component.VersionSpecification; +import com.yahoo.concurrent.CopyOnWriteHashMap; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.serialization.*; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Routable; + +import java.util.*; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This class encapsulates the logic required to map routable type and version to a corresponding {@link + * RoutableFactory}. It is owned and accessed through a {@link DocumentProtocol} instance. This class uses a factory + * cache to reduce the latency of matching version specifications to actual versions when resolving factories. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +final class RoutableRepository { + + private static final Logger log = Logger.getLogger(RoutableRepository.class.getName()); + private final CopyOnWriteHashMap<Integer, VersionMap> factoryTypes = new CopyOnWriteHashMap<>(); + private final CopyOnWriteHashMap<CacheKey, RoutableFactory> cache = new CopyOnWriteHashMap<>(); + private LoadTypeSet loadTypes; + + public RoutableRepository(LoadTypeSet set) { + loadTypes = set; + } + + /** + * Decodes a {@link Routable} from the given byte array. This uses the content of the byte array to dispatch the + * decode request to the appropriate {@link RoutableFactory} that was previously registered. + * + * If a routable can not be decoded, this method returns null. + * + * @param version The version of the encoded routable. + * @param data The byte array containing the encoded routable. + * @return The decoded routable. + */ + Routable decode(DocumentTypeManager docMan, Version version, byte[] data) { + if (data == null || data.length == 0) { + log.log(LogLevel.ERROR, "Received empty byte array for deserialization."); + return null; + } + DocumentDeserializer in; + + if (version.getMajor() >= 5) { + in = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data)); + } else { + in = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(data)); + } + + int type = in.getInt(null); + RoutableFactory factory = getFactory(version, type); + if (factory == null) { + log.log(LogLevel.ERROR, "No routable factory found for routable type " + type + + " (version " + version + ")."); + return null; + } + Routable ret = factory.decode(in, loadTypes); + if (ret == null) { + log.log(LogLevel.ERROR, "Routable factory " + factory.getClass().getName() + " failed to deserialize " + + "routable of type " + type + " (version " + version + ")."); + log.log(LogLevel.ERROR, Arrays.toString(data)); + return null; + } + return ret; + } + + /** + * Encodes a {@link Routable} into a byte array. This dispatches the encode request to the appropriate {@link + * RoutableFactory} that was previously registered. + * + * If a routable can not be encoded, this method returns an empty byte array. + * + * @param version The version to encode the routable as. + * @param obj The routable to encode. + * @return The byte array containing the encoded routable. + */ + byte[] encode(Version version, Routable obj) { + int type = obj.getType(); + RoutableFactory factory = getFactory(version, type); + if (factory == null) { + log.log(LogLevel.ERROR, "No routable factory found for routable type " + type + + " (version " + version + ")."); + return new byte[0]; + } + DocumentSerializer out; + + if (version.getMajor() >= 5) { + out = DocumentSerializerFactory.createHead(new GrowableByteBuffer(8192)); + } else { + out = DocumentSerializerFactory.create42(new GrowableByteBuffer(8192)); + } + + out.putInt(null, type); + if (!factory.encode(obj, out)) { + log.log(LogLevel.ERROR, "Routable factory " + factory.getClass().getName() + " failed to serialize " + + "routable of type " + type + " (version " + version + ")."); + return new byte[0]; + } + byte[] ret = new byte[out.getBuf().position()]; + out.getBuf().rewind(); + out.getBuf().get(ret); + return ret; + } + + /** + * Registers a routable factory for a given version and routable type. + * + * @param version The version specification that the given factory supports. + * @param type The routable type that the given factory supports. + * @param factory The routable factory to register. + */ + void putFactory(VersionSpecification version, int type, RoutableFactory factory) { + VersionMap versionMap = factoryTypes.get(type); + if (versionMap == null) { + versionMap = new VersionMap(); + + factoryTypes.put(type, versionMap); + } + if (versionMap.putFactory(version, factory)) { + cache.clear(); + } + } + + /** + * Returns the routable factory for a given version and routable type. + * + * @param version The version that the factory must support. + * @param type The routable type that the factory must support. + * @return The routable factory matching the criteria, or null. + */ + RoutableFactory getFactory(Version version, int type) { + CacheKey cacheKey = new CacheKey(version, type); + RoutableFactory factory = cache.get(cacheKey); + if (factory != null) { + return factory; + } + VersionMap versionMap = factoryTypes.get(type); + if (versionMap == null) { + return null; + } + factory = versionMap.getFactory(version); + if (factory == null) { + return null; + } + cache.put(cacheKey, factory); + return factory; + } + + /** + * Returns a list of routable types that support the given version. + * + * @param version The version to return types for. + * @return The list of supported types. + */ + List<Integer> getRoutableTypes(Version version) { + List<Integer> ret = new ArrayList<>(); + for (Map.Entry<Integer, VersionMap> entry : factoryTypes.entrySet()) { + if (entry.getValue().getFactory(version) != null) { + ret.add(entry.getKey()); + } + } + return ret; + } + + /** + * Internal helper class that implements a map from {@link VersionSpecification} to {@link RoutableFactory}. + */ + private static class VersionMap { + + final Map<VersionSpecification, RoutableFactory> factoryVersions = new HashMap<>(); + + boolean putFactory(VersionSpecification version, RoutableFactory factory) { + return factoryVersions.put(version, factory) == null; + } + + RoutableFactory getFactory(Version version) { + VersionSpecification versionSpec = version.toSpecification(); + + // Retrieve the factory with the highest version lower than or equal to actual version + return factoryVersions.entrySet().stream() + // Drop factories that have a higher version than actual version + .filter(entry -> entry.getKey().compareTo(versionSpec) <= 0) + + // Get the factory with the highest version + .max((entry1, entry2) -> entry1.getKey().compareTo(entry2.getKey())) + .map(Map.Entry::getValue) + + // Return factory or null if no suitable factory found + .orElse(null); + } + } + + /** + * Internal helper class that implements a cache key for mapping a {@link Version} and routable type to a {@link + * RoutableFactory}. + */ + private static class CacheKey { + + final Version version; + final int type; + + public CacheKey(Version version, int type) { + this.version = version; + this.type = type; + } + + @Override + public int hashCode() { + return version.hashCode() + type; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof CacheKey)) { + return false; + } + CacheKey rhs = (CacheKey)obj; + if (!version.equals(rhs.version)) { + return false; + } + if (type != rhs.type) { + return false; + } + return true; + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java index 05e39503308..beb295509d1 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java @@ -1,148 +1,148 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public abstract class RoutingPolicyFactories {
-
- static class AndPolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new ANDPolicy(param);
- }
-
-
- public void destroy() {
- }
- }
-
- static class StoragePolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new StoragePolicy(param);
- }
-
- public void destroy() {
- }
- }
-
- static class ContentPolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new ContentPolicy(param);
- }
-
- public void destroy() {
- }
- }
-
- static class MessageTypePolicyFactory implements RoutingPolicyFactory {
- private final String configId;
-
- public MessageTypePolicyFactory(String configId) {
- this.configId = configId;
- }
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new MessageTypePolicy((param == null || param.isEmpty()) ? configId : param);
- }
-
- public void destroy() {
- }
- }
-
- static class DocumentRouteSelectorPolicyFactory implements RoutingPolicyFactory {
-
- private final String configId;
-
- public DocumentRouteSelectorPolicyFactory(String configId) {
- this.configId = configId;
- }
-
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- DocumentRouteSelectorPolicy ret = new DocumentRouteSelectorPolicy((param == null || param.isEmpty()) ?
- configId : param);
- String error = ret.getError();
- if (error != null) {
- return new ErrorPolicy(error);
- }
- return ret;
- }
-
-
- public void destroy() {
- }
- }
-
- static class ExternPolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- ExternPolicy ret = new ExternPolicy(param);
- String error = ret.getError();
- if (error != null) {
- return new ErrorPolicy(error);
- }
- return ret;
- }
-
-
- public void destroy() {
- }
- }
-
- static class LocalServicePolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new LocalServicePolicy(param);
- }
-
-
- public void destroy() {
- }
- }
-
- static class RoundRobinPolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new RoundRobinPolicy();
- }
-
-
- public void destroy() {
- }
- }
-
- static class LoadBalancerPolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new LoadBalancerPolicy(param);
- }
-
-
- public void destroy() {
- }
- }
-
- static class SearchColumnPolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new SearchColumnPolicy(param);
- }
-
-
- public void destroy() {
- }
- }
-
- static class SearchRowPolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new SearchRowPolicy(param);
- }
-
-
- public void destroy() {
- }
- }
-
- static class SubsetServicePolicyFactory implements RoutingPolicyFactory {
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new SubsetServicePolicy(param);
- }
-
-
- public void destroy() {
- }
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class RoutingPolicyFactories { + + static class AndPolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new ANDPolicy(param); + } + + + public void destroy() { + } + } + + static class StoragePolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new StoragePolicy(param); + } + + public void destroy() { + } + } + + static class ContentPolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new ContentPolicy(param); + } + + public void destroy() { + } + } + + static class MessageTypePolicyFactory implements RoutingPolicyFactory { + private final String configId; + + public MessageTypePolicyFactory(String configId) { + this.configId = configId; + } + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new MessageTypePolicy((param == null || param.isEmpty()) ? configId : param); + } + + public void destroy() { + } + } + + static class DocumentRouteSelectorPolicyFactory implements RoutingPolicyFactory { + + private final String configId; + + public DocumentRouteSelectorPolicyFactory(String configId) { + this.configId = configId; + } + + public DocumentProtocolRoutingPolicy createPolicy(String param) { + DocumentRouteSelectorPolicy ret = new DocumentRouteSelectorPolicy((param == null || param.isEmpty()) ? + configId : param); + String error = ret.getError(); + if (error != null) { + return new ErrorPolicy(error); + } + return ret; + } + + + public void destroy() { + } + } + + static class ExternPolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + ExternPolicy ret = new ExternPolicy(param); + String error = ret.getError(); + if (error != null) { + return new ErrorPolicy(error); + } + return ret; + } + + + public void destroy() { + } + } + + static class LocalServicePolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new LocalServicePolicy(param); + } + + + public void destroy() { + } + } + + static class RoundRobinPolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new RoundRobinPolicy(); + } + + + public void destroy() { + } + } + + static class LoadBalancerPolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new LoadBalancerPolicy(param); + } + + + public void destroy() { + } + } + + static class SearchColumnPolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new SearchColumnPolicy(param); + } + + + public void destroy() { + } + } + + static class SearchRowPolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new SearchRowPolicy(param); + } + + + public void destroy() { + } + } + + static class SubsetServicePolicyFactory implements RoutingPolicyFactory { + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new SubsetServicePolicy(param); + } + + + public void destroy() { + } + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java index 3bfa85ac4d5..26b0eec8cf4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java @@ -1,76 +1,76 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.documentapi.metrics.DocumentProtocolMetricSet;
-import com.yahoo.messagebus.routing.RoutingPolicy;
-import com.yahoo.log.LogLevel;
-
-import java.util.Map;
-import java.util.logging.Logger;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-class RoutingPolicyRepository {
-
- private static final Logger log = Logger.getLogger(RoutingPolicyRepository.class.getName());
- private final Map<String, RoutingPolicyFactory> factories = new ConcurrentHashMap<String, RoutingPolicyFactory>();
- private final DocumentProtocolMetricSet metrics;
-
- RoutingPolicyRepository(DocumentProtocolMetricSet metrics) {
- this.metrics = metrics;
- }
-
- /**
- * Registers a routing policy factory for a given name.
- *
- * @param name The name of the factory to register.
- * @param factory The factory to register.
- */
- void putFactory(String name, RoutingPolicyFactory factory) {
- factories.put(name, factory);
- }
-
- /**
- * Returns the routing policy factory for a given name.
- *
- * @param name The name of the factory to return.
- * @return The routing policy factory matching the criteria, or null.
- */
- RoutingPolicyFactory getFactory(String name) {
- return factories.get(name);
- }
-
- /**
- * Creates and returns a routing policy using the named factory and the given parameter.
- *
- * @param name The name of the factory to use.
- * @param param The parameter to pass to the factory.
- * @return The created policy.
- */
- RoutingPolicy createPolicy(String name, String param) {
- RoutingPolicyFactory factory = getFactory(name);
- if (factory == null) {
- log.log(LogLevel.ERROR, "No routing policy factory found for name '" + name + "'.");
- return null;
- }
- DocumentProtocolRoutingPolicy ret;
- try {
- ret = factory.createPolicy(param);
- } catch (Exception e) {
- ret = new ErrorPolicy(e.getMessage());
- }
-
- if (ret.getMetrics() != null) {
- metrics.routingPolicyMetrics.addMetric(ret.getMetrics());
- }
-
- if (ret == null) {
- log.log(LogLevel.ERROR, "Routing policy factory " + factory.getClass().getName() + " failed to create a " +
- "routing policy for parameter '" + name + "'.");
- return null;
- }
- return ret;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.documentapi.metrics.DocumentProtocolMetricSet; +import com.yahoo.messagebus.routing.RoutingPolicy; +import com.yahoo.log.LogLevel; + +import java.util.Map; +import java.util.logging.Logger; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +class RoutingPolicyRepository { + + private static final Logger log = Logger.getLogger(RoutingPolicyRepository.class.getName()); + private final Map<String, RoutingPolicyFactory> factories = new ConcurrentHashMap<String, RoutingPolicyFactory>(); + private final DocumentProtocolMetricSet metrics; + + RoutingPolicyRepository(DocumentProtocolMetricSet metrics) { + this.metrics = metrics; + } + + /** + * Registers a routing policy factory for a given name. + * + * @param name The name of the factory to register. + * @param factory The factory to register. + */ + void putFactory(String name, RoutingPolicyFactory factory) { + factories.put(name, factory); + } + + /** + * Returns the routing policy factory for a given name. + * + * @param name The name of the factory to return. + * @return The routing policy factory matching the criteria, or null. + */ + RoutingPolicyFactory getFactory(String name) { + return factories.get(name); + } + + /** + * Creates and returns a routing policy using the named factory and the given parameter. + * + * @param name The name of the factory to use. + * @param param The parameter to pass to the factory. + * @return The created policy. + */ + RoutingPolicy createPolicy(String name, String param) { + RoutingPolicyFactory factory = getFactory(name); + if (factory == null) { + log.log(LogLevel.ERROR, "No routing policy factory found for name '" + name + "'."); + return null; + } + DocumentProtocolRoutingPolicy ret; + try { + ret = factory.createPolicy(param); + } catch (Exception e) { + ret = new ErrorPolicy(e.getMessage()); + } + + if (ret.getMetrics() != null) { + metrics.routingPolicyMetrics.addMetric(ret.getMetrics()); + } + + if (ret == null) { + log.log(LogLevel.ERROR, "Routing policy factory " + factory.getClass().getName() + " failed to create a " + + "routing policy for parameter '" + name + "'."); + return null; + } + return ret; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchRowPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchRowPolicy.java index d36d3ee1e4c..d6c644b238c 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchRowPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SearchRowPolicy.java @@ -1,85 +1,85 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.log.LogLevel;
-import com.yahoo.messagebus.ErrorCode;
-import com.yahoo.messagebus.Reply;
-import com.yahoo.messagebus.metrics.MetricSet;
-import com.yahoo.messagebus.routing.RoutingContext;
-import com.yahoo.messagebus.routing.RoutingNodeIterator;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Logger;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class SearchRowPolicy implements DocumentProtocolRoutingPolicy {
-
- private static Logger log = Logger.getLogger(SearchRowPolicy.class.getName());
- private int minOk = 0; // Hide OUT_OF_SERVICE as long as this number of replies are something else.
-
- /**
- * Creates a search row policy that wraps the underlying search group policy in case the parameter is something
- * other than an empty string.
- *
- * @param param The number of minimum non-OOS replies that this policy requires.
- */
- public SearchRowPolicy(String param) {
- if (param != null && param.length() > 0) {
- try {
- minOk = Integer.parseInt(param);
- }
- catch (NumberFormatException e) {
- log.log(LogLevel.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e);
- }
- if (minOk <= 0) {
- log.log(LogLevel.WARNING, "Ignoring a request to set the minimum number of OK replies to " + minOk + " " +
- "because it makes no sense. This routing policy will not allow any recipient " +
- "to be out of service.");
- }
- }
- }
-
- @Override
- public void select(RoutingContext context) {
- context.addChildren(context.getMatchedRecipients());
- context.setSelectOnRetry(false);
- if (minOk > 0) {
- context.addConsumableError(ErrorCode.SERVICE_OOS);
- }
- }
-
- @Override
- public void merge(RoutingContext context) {
- if (minOk > 0) {
- Set<Integer> oosReplies = new HashSet<Integer>();
- int idx = 0;
- for (RoutingNodeIterator it = context.getChildIterator();
- it.isValid(); it.next())
- {
- Reply ref = it.getReplyRef();
- if (ref.hasErrors() && DocumentProtocol.hasOnlyErrorsOfType(ref, ErrorCode.SERVICE_OOS)) {
- oosReplies.add(idx);
- }
- ++idx;
- }
- if (context.getNumChildren() - oosReplies.size() >= minOk) {
- DocumentProtocol.merge(context, oosReplies);
- return;
- }
- }
- DocumentProtocol.merge(context);
- }
-
- @Override
- public void destroy() {
- // empty
- }
-
- @Override
- public MetricSet getMetrics() {
- return null;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.RoutingContext; +import com.yahoo.messagebus.routing.RoutingNodeIterator; + +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SearchRowPolicy implements DocumentProtocolRoutingPolicy { + + private static Logger log = Logger.getLogger(SearchRowPolicy.class.getName()); + private int minOk = 0; // Hide OUT_OF_SERVICE as long as this number of replies are something else. + + /** + * Creates a search row policy that wraps the underlying search group policy in case the parameter is something + * other than an empty string. + * + * @param param The number of minimum non-OOS replies that this policy requires. + */ + public SearchRowPolicy(String param) { + if (param != null && param.length() > 0) { + try { + minOk = Integer.parseInt(param); + } + catch (NumberFormatException e) { + log.log(LogLevel.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e); + } + if (minOk <= 0) { + log.log(LogLevel.WARNING, "Ignoring a request to set the minimum number of OK replies to " + minOk + " " + + "because it makes no sense. This routing policy will not allow any recipient " + + "to be out of service."); + } + } + } + + @Override + public void select(RoutingContext context) { + context.addChildren(context.getMatchedRecipients()); + context.setSelectOnRetry(false); + if (minOk > 0) { + context.addConsumableError(ErrorCode.SERVICE_OOS); + } + } + + @Override + public void merge(RoutingContext context) { + if (minOk > 0) { + Set<Integer> oosReplies = new HashSet<Integer>(); + int idx = 0; + for (RoutingNodeIterator it = context.getChildIterator(); + it.isValid(); it.next()) + { + Reply ref = it.getReplyRef(); + if (ref.hasErrors() && DocumentProtocol.hasOnlyErrorsOfType(ref, ErrorCode.SERVICE_OOS)) { + oosReplies.add(idx); + } + ++idx; + } + if (context.getNumChildren() - oosReplies.size() >= minOk) { + DocumentProtocol.merge(context, oosReplies); + return; + } + } + DocumentProtocol.merge(context); + } + + @Override + public void destroy() { + // empty + } + + @Override + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketMessage.java index 3854637ba5f..615699c674e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketMessage.java @@ -1,60 +1,60 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.document.BucketId;
-
-public class StatBucketMessage extends DocumentMessage {
-
- private BucketId bucketId;
- private String documentSelection;
-
- StatBucketMessage() {
- // need to deserialize into
- }
-
- public StatBucketMessage(BucketId bucket, String documentSelection) {
- this.bucketId = bucket;
- this.documentSelection = documentSelection;
- }
-
- public BucketId getBucketId() {
- return bucketId;
- }
-
- void setBucketId(BucketId bucket) {
- bucketId = bucket;
- }
-
- public String getDocumentSelection() {
- return documentSelection;
- }
-
- void setDocumentSelection(String documentSelection) {
- this.documentSelection = documentSelection;
- }
-
- @Override
- public DocumentReply createReply() {
- return new StatBucketReply();
- }
-
- @Override
- public int getApproxSize() {
- return super.getApproxSize() + 8 + documentSelection.length();
- }
-
- @Override
- public boolean hasSequenceId() {
- return true;
- }
-
- @Override
- public long getSequenceId() {
- return bucketId.getRawId();
- }
-
- @Override
- public int getType() {
- return DocumentProtocol.MESSAGE_STATBUCKET;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.document.BucketId; + +public class StatBucketMessage extends DocumentMessage { + + private BucketId bucketId; + private String documentSelection; + + StatBucketMessage() { + // need to deserialize into + } + + public StatBucketMessage(BucketId bucket, String documentSelection) { + this.bucketId = bucket; + this.documentSelection = documentSelection; + } + + public BucketId getBucketId() { + return bucketId; + } + + void setBucketId(BucketId bucket) { + bucketId = bucket; + } + + public String getDocumentSelection() { + return documentSelection; + } + + void setDocumentSelection(String documentSelection) { + this.documentSelection = documentSelection; + } + + @Override + public DocumentReply createReply() { + return new StatBucketReply(); + } + + @Override + public int getApproxSize() { + return super.getApproxSize() + 8 + documentSelection.length(); + } + + @Override + public boolean hasSequenceId() { + return true; + } + + @Override + public long getSequenceId() { + return bucketId.getRawId(); + } + + @Override + public int getType() { + return DocumentProtocol.MESSAGE_STATBUCKET; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketReply.java index 43c369106d1..a1439ef845f 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketReply.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StatBucketReply.java @@ -1,19 +1,19 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-public class StatBucketReply extends DocumentReply {
-
- private String results = "";
-
- public StatBucketReply() {
- super(DocumentProtocol.REPLY_STATBUCKET);
- }
-
- public String getResults() {
- return results;
- }
-
- public void setResults(String result) {
- results = result;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +public class StatBucketReply extends DocumentReply { + + private String results = ""; + + public StatBucketReply() { + super(DocumentProtocol.REPLY_STATBUCKET); + } + + public String getResults() { + return results; + } + + public void setResults(String result) { + results = result; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java index dc06fe7042d..76e74b98f86 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java @@ -1,145 +1,145 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.jrt.slobrok.api.Mirror;
-import com.yahoo.log.LogLevel;
-import com.yahoo.messagebus.metrics.MetricSet;
-import com.yahoo.messagebus.routing.*;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Logger;
-
-/**
- * This policy implements the logic to select a subset of services that matches a slobrok pattern.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class SubsetServicePolicy implements DocumentProtocolRoutingPolicy {
-
- private static Logger log = Logger.getLogger(SubsetServicePolicy.class.getName());
- private final int subsetSize;
- private final Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>();
-
- /**
- * Creates an instance of a subset service policy. The parameter string is parsed as an integer number that is the
- * number of services to include in the set to choose from.
- *
- * @param param The number of services to include in the set.
- */
- public SubsetServicePolicy(String param) {
- int subsetSize = 5;
- if (param != null && param.length() > 0) {
- try {
- subsetSize = Integer.parseInt(param);
- }
- catch (NumberFormatException e) {
- log.log(LogLevel.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e);
- }
- if (subsetSize <= 0) {
- log.warning("Ignoring a request to set the subset size to " + subsetSize + " because it makes no " +
- "sense. This routing policy will choose any one matching service.");
- }
- } else {
- log.warning("No parameter given to SubsetService policy, using default value " + subsetSize + ".");
- }
- this.subsetSize = subsetSize;
- }
-
- // Inherit doc from RoutingPolicy.
- public void select(RoutingContext ctx) {
- Route route = new Route(ctx.getRoute());
- route.setHop(0, getRecipient(ctx));
- ctx.addChild(route);
- }
-
- // Inherit doc from RoutingPolicy.
- public void merge(RoutingContext ctx) {
- DocumentProtocol.merge(ctx);
- }
-
- /**
- * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to
- * the internal cache.
- *
- * @param ctx The routing context.
- * @return The recipient hop to use.
- */
- private Hop getRecipient(RoutingContext ctx) {
- Hop hop = null;
- if (subsetSize > 0) {
- synchronized (this) {
- CacheEntry entry = update(ctx);
- if (!entry.recipients.isEmpty()) {
- if (++entry.offset >= entry.recipients.size()) {
- entry.offset = 0;
- }
- hop = new Hop(entry.recipients.get(entry.offset));
- }
- }
- }
- if (hop == null) {
- hop = new Hop(ctx.getRoute().getHop(0));
- hop.setDirective(ctx.getDirectiveIndex(), new VerbatimDirective("*"));
- }
- return hop;
- }
-
- /**
- * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is
- * handled outside of it.
- *
- * @param ctx The routing context.
- * @return The updated cache entry.
- */
- private CacheEntry update(RoutingContext ctx) {
- String key = getCacheKey(ctx);
- CacheEntry entry = cache.get(key);
- if (entry == null) {
- entry = new CacheEntry();
- cache.put(key, entry);
- }
-
- int upd = ctx.getMirror().updates();
- if (entry.generation != upd) {
- entry.generation = upd;
- entry.recipients.clear();
-
- Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix());
- int pos = ctx.getMessageBus().getConnectionSpec().hashCode();
- for (int i = 0; i < subsetSize && i < arr.length; ++i) {
- entry.recipients.add(Hop.parse(arr[((pos + i) & Integer.MAX_VALUE) % arr.length].getName()));
- }
- }
- return entry;
- }
-
- /**
- * Returns a cache key for this instance of the policy. Because behaviour is based on the hop in which the policy
- * occurs, the cache key is the hop string itself.
- *
- * @param ctx The routing context.
- * @return The cache key.
- */
- private String getCacheKey(RoutingContext ctx) {
- return ctx.getRoute().getHop(0).toString();
- }
-
- /**
- * Defines the necessary cache data.
- */
- private class CacheEntry {
- private final List<Hop> recipients = new ArrayList<Hop>();
- private int generation = 0;
- private int offset = 0;
- }
-
- public void destroy() {
- }
-
- public MetricSet getMetrics() {
- return null;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.routing.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +/** + * This policy implements the logic to select a subset of services that matches a slobrok pattern. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SubsetServicePolicy implements DocumentProtocolRoutingPolicy { + + private static Logger log = Logger.getLogger(SubsetServicePolicy.class.getName()); + private final int subsetSize; + private final Map<String, CacheEntry> cache = new HashMap<String, CacheEntry>(); + + /** + * Creates an instance of a subset service policy. The parameter string is parsed as an integer number that is the + * number of services to include in the set to choose from. + * + * @param param The number of services to include in the set. + */ + public SubsetServicePolicy(String param) { + int subsetSize = 5; + if (param != null && param.length() > 0) { + try { + subsetSize = Integer.parseInt(param); + } + catch (NumberFormatException e) { + log.log(LogLevel.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e); + } + if (subsetSize <= 0) { + log.warning("Ignoring a request to set the subset size to " + subsetSize + " because it makes no " + + "sense. This routing policy will choose any one matching service."); + } + } else { + log.warning("No parameter given to SubsetService policy, using default value " + subsetSize + "."); + } + this.subsetSize = subsetSize; + } + + // Inherit doc from RoutingPolicy. + public void select(RoutingContext ctx) { + Route route = new Route(ctx.getRoute()); + route.setHop(0, getRecipient(ctx)); + ctx.addChild(route); + } + + // Inherit doc from RoutingPolicy. + public void merge(RoutingContext ctx) { + DocumentProtocol.merge(ctx); + } + + /** + * Returns the appropriate recipient hop for the given routing context. This method provides synchronized access to + * the internal cache. + * + * @param ctx The routing context. + * @return The recipient hop to use. + */ + private Hop getRecipient(RoutingContext ctx) { + Hop hop = null; + if (subsetSize > 0) { + synchronized (this) { + CacheEntry entry = update(ctx); + if (!entry.recipients.isEmpty()) { + if (++entry.offset >= entry.recipients.size()) { + entry.offset = 0; + } + hop = new Hop(entry.recipients.get(entry.offset)); + } + } + } + if (hop == null) { + hop = new Hop(ctx.getRoute().getHop(0)); + hop.setDirective(ctx.getDirectiveIndex(), new VerbatimDirective("*")); + } + return hop; + } + + /** + * Updates and returns the cache entry for the given routing context. This method assumes that synchronization is + * handled outside of it. + * + * @param ctx The routing context. + * @return The updated cache entry. + */ + private CacheEntry update(RoutingContext ctx) { + String key = getCacheKey(ctx); + CacheEntry entry = cache.get(key); + if (entry == null) { + entry = new CacheEntry(); + cache.put(key, entry); + } + + int upd = ctx.getMirror().updates(); + if (entry.generation != upd) { + entry.generation = upd; + entry.recipients.clear(); + + Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); + int pos = ctx.getMessageBus().getConnectionSpec().hashCode(); + for (int i = 0; i < subsetSize && i < arr.length; ++i) { + entry.recipients.add(Hop.parse(arr[((pos + i) & Integer.MAX_VALUE) % arr.length].getName())); + } + } + return entry; + } + + /** + * Returns a cache key for this instance of the policy. Because behaviour is based on the hop in which the policy + * occurs, the cache key is the hop string itself. + * + * @param ctx The routing context. + * @return The cache key. + */ + private String getCacheKey(RoutingContext ctx) { + return ctx.getRoute().getHop(0).toString(); + } + + /** + * Defines the necessary cache data. + */ + private class CacheEntry { + private final List<Hop> recipients = new ArrayList<Hop>(); + private int generation = 0; + private int offset = 0; + } + + public void destroy() { + } + + public MetricSet getMetrics() { + return null; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentReply.java index 5f091101554..30801cbbac0 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentReply.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentReply.java @@ -1,35 +1,35 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class UpdateDocumentReply extends WriteDocumentReply {
-
- private boolean found = true;
-
- /**
- * Constructs a new reply with no content.
- */
- public UpdateDocumentReply() {
- super(DocumentProtocol.REPLY_UPDATEDOCUMENT);
- }
-
- /**
- * Returns whether or not the document was found and updated.
- *
- * @return true if document was found
- */
- public boolean wasFound() {
- return found;
- }
-
- /**
- * Sets whether or not the document was found and updated.
- *
- * @param found True if the document was found
- */
- public void setWasFound(boolean found) {
- this.found = found;
- }
-}
\ No newline at end of file +package com.yahoo.documentapi.messagebus.protocol; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class UpdateDocumentReply extends WriteDocumentReply { + + private boolean found = true; + + /** + * Constructs a new reply with no content. + */ + public UpdateDocumentReply() { + super(DocumentProtocol.REPLY_UPDATEDOCUMENT); + } + + /** + * Returns whether or not the document was found and updated. + * + * @return true if document was found + */ + public boolean wasFound() { + return found; + } + + /** + * Sets whether or not the document was found and updated. + * + * @param found True if the document was found + */ + public void setWasFound(boolean found) { + this.found = found; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Argument.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Argument.java index 3a434eab101..e60ff35bdcf 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Argument.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Argument.java @@ -1,40 +1,40 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.systemstate.rule;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class Argument {
-
- private final String name;
- private final String value;
-
- /**
- * Constructs a new argument.
- *
- * @param name The name of this argument.
- * @param value The value of this argument.
- */
- public Argument(String name, String value) {
- this.name = name;
- this.value = value;
- }
-
- /**
- * Returns the name of this argument.
- *
- * @return The name.
- */
- public String getName() {
- return name;
- }
-
- /**
- * Returns the value of this argument.
- *
- * @return The value.
- */
- public String getValue() {
- return value;
- }
-}
+package com.yahoo.documentapi.messagebus.systemstate.rule; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Argument { + + private final String name; + private final String value; + + /** + * Constructs a new argument. + * + * @param name The name of this argument. + * @param value The value of this argument. + */ + public Argument(String name, String value) { + this.name = name; + this.value = value; + } + + /** + * Returns the name of this argument. + * + * @return The name. + */ + public String getName() { + return name; + } + + /** + * Returns the value of this argument. + * + * @return The value. + */ + public String getValue() { + return value; + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java index 870a39c0122..3a8fe5c6229 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/Location.java @@ -1,120 +1,120 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.systemstate.rule;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Arrays;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class Location {
-
- private List<String> items = new ArrayList<String>();
-
- /**
- * Constructs a new location with no items.
- */
- public Location() {
- // empty
- }
-
- /**
- * Constructs a new location based on a location string.
- *
- * @param loc The location string to parse.
- */
- public Location(String loc) {
- items.addAll(Arrays.asList(loc.split("/")));
- normalize();
- }
-
- /**
- * Constructs a new location based on a list of items.
- *
- * @param items The components that make up this location.
- */
- public Location(List<String> items) {
- this.items.addAll(items);
- normalize();
- }
-
- /**
- * Constructs a new location as a copy of another.
- *
- * @param loc The location to copy.
- */
- public Location(Location loc) {
- items.addAll(loc.items);
- }
-
- /**
- * Constructs a new location based on a working directory and a list of items.
- *
- * @param pwd The path of the working directory.
- * @param items The components that make up this location.
- */
- public Location(Location pwd, List<String> items) {
- this.items.addAll(pwd.getItems());
- this.items.addAll(items);
- normalize();
- }
-
- /**
- * Returns a location object that represents the "next" step along this location path. This means removing the first
- * elements of this location's items and returning a new location for this sublist.
- *
- * @return The next location along this path.
- */
- public Location getNext() {
- List<String> next = new ArrayList<String>(items);
- next.remove(0);
- return new Location(next);
- }
-
- /**
- * Returns the components of this location.
- *
- * @return The component array.
- */
- public List<String> getItems() {
- return items;
- }
-
- /**
- * Normalizes the items list of this location so that all PREV or THIS locations are replaced by their actual
- * meaning. This carries some overhead since it is not done in place.
- *
- * @return This, to allow chaining.
- */
- private Location normalize() {
- List<String> norm = new ArrayList<String>();
- for (String item : items) {
- if (item.equals(NodeState.NODE_PARENT)) {
- if (norm.size() == 0) {
- // ignore
- }
- else {
- norm.remove(norm.size() - 1);
- }
- }
- else if (!item.equals(NodeState.NODE_CURRENT)) {
- norm.add(item);
- }
- }
- items = norm;
- return this;
- }
-
- @Override
- public String toString() {
- StringBuffer ret = new StringBuffer();
- for (int i = 0; i < items.size(); ++i) {
- ret.append(items.get(i));
- if (i < items.size() - 1) {
- ret.append("/");
- }
- }
- return ret.toString();
- }
-}
+package com.yahoo.documentapi.messagebus.systemstate.rule; + +import java.util.ArrayList; +import java.util.List; +import java.util.Arrays; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Location { + + private List<String> items = new ArrayList<String>(); + + /** + * Constructs a new location with no items. + */ + public Location() { + // empty + } + + /** + * Constructs a new location based on a location string. + * + * @param loc The location string to parse. + */ + public Location(String loc) { + items.addAll(Arrays.asList(loc.split("/"))); + normalize(); + } + + /** + * Constructs a new location based on a list of items. + * + * @param items The components that make up this location. + */ + public Location(List<String> items) { + this.items.addAll(items); + normalize(); + } + + /** + * Constructs a new location as a copy of another. + * + * @param loc The location to copy. + */ + public Location(Location loc) { + items.addAll(loc.items); + } + + /** + * Constructs a new location based on a working directory and a list of items. + * + * @param pwd The path of the working directory. + * @param items The components that make up this location. + */ + public Location(Location pwd, List<String> items) { + this.items.addAll(pwd.getItems()); + this.items.addAll(items); + normalize(); + } + + /** + * Returns a location object that represents the "next" step along this location path. This means removing the first + * elements of this location's items and returning a new location for this sublist. + * + * @return The next location along this path. + */ + public Location getNext() { + List<String> next = new ArrayList<String>(items); + next.remove(0); + return new Location(next); + } + + /** + * Returns the components of this location. + * + * @return The component array. + */ + public List<String> getItems() { + return items; + } + + /** + * Normalizes the items list of this location so that all PREV or THIS locations are replaced by their actual + * meaning. This carries some overhead since it is not done in place. + * + * @return This, to allow chaining. + */ + private Location normalize() { + List<String> norm = new ArrayList<String>(); + for (String item : items) { + if (item.equals(NodeState.NODE_PARENT)) { + if (norm.size() == 0) { + // ignore + } + else { + norm.remove(norm.size() - 1); + } + } + else if (!item.equals(NodeState.NODE_CURRENT)) { + norm.add(item); + } + } + items = norm; + return this; + } + + @Override + public String toString() { + StringBuffer ret = new StringBuffer(); + for (int i = 0; i < items.size(); ++i) { + ret.append(items.get(i)); + if (i < items.size() - 1) { + ret.append("/"); + } + } + return ret.toString(); + } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java index f5920f32119..ccd5c0c811d 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java @@ -1,310 +1,310 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.systemstate.rule;
-
-import com.yahoo.log.LogLevel;
-
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Logger;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class NodeState {
-
- /** A location string that expresses the use of the PARENT node. */
- public static final String NODE_PARENT = "..";
-
- /** A location string that expresses the use of THIS node. */
- public static final String NODE_CURRENT = ".";
-
- private static Logger log = Logger.getLogger(NodeState.class.getName());
- private final Map<String, NodeState> children = new LinkedHashMap<String, NodeState>();
- private final Map<String, String> state = new LinkedHashMap<String, String>();
- private NodeState parent = null;
- private String id = null;
-
- /**
- * Creates a node state that no internal content.
- */
- public NodeState() {
- // empty
- }
-
- /**
- * Creates a node state based on a list of argument objects. These arguments are iterated and added to this node's
- * internal state map.
- *
- * @param args The arguments to use as state.
- */
- public NodeState(List<Argument> args) {
- for (Argument arg : args) {
- setState(arg.getName(), arg.getValue());
- }
- }
-
- /**
- * Adds a child to this node at the given location. The key can be a location string, in which case the necessary
- * intermediate node states are created.
- *
- * @param key The location at which to add the child.
- * @param child The child node to add.
- * @return This, to allow chaining.
- */
- public NodeState addChild(String key, NodeState child) {
- getChild(key, true).copy(child);
- return this;
- }
-
- /**
- * Returns the child at the given location relative to this.
- *
- * @param key The location of the child to return.
- * @return The child object, null if not found.
- */
- public NodeState getChild(String key) {
- return getChild(key, false);
- }
-
- /**
- * Returns the child at the given location relative to this. This method can be forced to return a child node even
- * if it does not exist, by adding all intermediate nodes and the target node itself.
- *
- * @param key The location of the child to return.
- * @param force Whether or not to force a return value by creating missing nodes.
- * @return The child object, null if not found.
- */
- public NodeState getChild(String key, boolean force) {
- if (key == null || key.length() == 0) {
- return this;
- }
- String arr[] = key.split("/", 2);
- while (arr.length == 2 && arr[0].equals(NODE_CURRENT)) {
- arr = arr[1].split("/", 2);
- }
- if (arr[0].equals(NODE_CURRENT)) {
- return this;
- }
- if (arr[0].equals(NODE_PARENT)) {
- if (parent == null) {
- log.log(LogLevel.ERROR, "Location string '" + key + "' requests a parent above the top-most node, " +
- "returning self to avoid crash.");
- }
- return parent.getChild(arr[1], force);
- }
- if (!children.containsKey(arr[0])) {
- if (!force) {
- return null;
- }
- children.put(arr[0], new NodeState());
- children.get(arr[0]).setParent(this, arr[0]);
- }
- if (arr.length == 2) {
- return children.get(arr[0]).getChild(arr[1], force);
- }
- return children.get(arr[0]);
- }
-
- /**
- * Returns the map of child nodes for iteration.
- *
- * @return The internal child map.
- */
- public Map<String, NodeState> getChildren() {
- return children;
- }
-
- /**
- * Removes the named child node from this node, and attempts to compact the system state from this node upwards by
- * removing empty nodes.
- *
- * @param key The child to remove.
- * @return The result of invoking {@link #compact} after the remove.
- */
- public NodeState removeChild(String key) {
- if (key == null || key.length() == 0) {
- return this;
- }
- int pos = key.lastIndexOf("/");
- if (pos > -1) {
- NodeState parent = getChild(key.substring(0, pos), false);
- if (parent != null) {
- return parent.removeChild(key.substring(pos + 1));
- }
- }
- else {
- children.remove(key);
- }
- return compact();
- }
-
- /**
- * Retrieves some arbitrary state information for a given key. The key can be a location string, in which case the
- * necessary intermediate nodes are traversed. If the key is not found, this method returns null.
- *
- * @param key The name of the state information to return.
- * @return The value of the state key.
- */
- public String getState(String key) {
- if (key == null || key.length() == 0) {
- return null;
- }
- int pos = key.lastIndexOf("/");
- if (pos > -1) {
- NodeState parent = getChild(key.substring(0, pos), false);
- return parent != null ? parent.getState(key.substring(pos + 1)) : null;
- }
- return state.get(key);
- }
-
- /**
- * Sets some arbitrary state data in this node. The key can be a location string, in which case the necessary
- * intermediate nodes are traversed and even created if missing.
- *
- * @param key The key to set.
- * @param value The value to assign to the key.
- * @return This, to allow chaining.
- */
- public NodeState setState(String key, String value) {
- if (key == null || key.length() == 0) {
- return this;
- }
- int pos = key.lastIndexOf("/");
- if (pos > -1) {
- getChild(key.substring(0, pos), true).setState(key.substring(pos + 1), value);
- }
- else {
- if (value == null || value.length() == 0) {
- return removeState(key);
- }
- else {
- state.put(key, value);
- }
- }
- return this;
- }
-
- /**
- * Removes the named (key, value) state pair from this node, and attempts to compact the system state from this node
- * upwards by removing empty nodes.
- *
- * @param key The state variable to clear.
- * @return The result of invoking {@link #compact} after the remove.
- */
- public NodeState removeState(String key) {
- if (key == null || key.length() == 0) {
- return this;
- }
- int pos = key.lastIndexOf("/");
- if (pos > -1) {
- NodeState parent = getChild(key.substring(0, pos), false);
- if (parent != null) {
- return parent.removeState(key.substring(pos + 1));
- }
- }
- else {
- state.remove(key);
- }
- return compact();
- }
-
- /**
- * Compacts the system state tree from this node upwards. This will delete itself if it has a parent, but no
- * internal state and no children.
- *
- * @return This or the first non-null ancestor, to allow chaining.
- */
- private NodeState compact() {
- if (state.isEmpty() && children.isEmpty()) {
- if (parent != null) {
- return parent.removeChild(id);
- }
- }
- return this;
- }
-
- /**
- * Copies the state content of another node state object into this.
- *
- * @param node The node state to copy into this.
- * @return This, to allow chaining.
- */
- public NodeState copy(NodeState node) {
- for (String key : node.state.keySet()) {
- state.put(key, node.state.get(key));
- }
- for (String key : node.children.keySet()) {
- getChild(key, true).copy(node.children.get(key));
- }
- return this;
- }
-
- /**
- * Clears both the internal state and child list, then compacts the tree from this node upwards.
- *
- * @return The result of invoking {@link #compact} after the remove.
- */
- public NodeState clear() {
- state.clear();
- children.clear();
- return compact();
- }
-
- /**
- * Sets the parent of this node.
- *
- * @param parent The parent node.
- * @param id The identifier of this node as seen in the parent.
- * @return This, to allow chaining.
- */
- public NodeState setParent(NodeState parent, String id) {
- this.parent = parent;
- this.id = id;
- return this;
- }
-
- /**
- * Returns a string representation of this node state.
- *
- * @param prefix The prefix to use for this string.
- * @return A string representation of this.
- * @throws UnsupportedEncodingException Thrown if the host system does not support UTF-8 encoding.
- */
- private String toString(String prefix) throws UnsupportedEncodingException {
- StringBuffer buf = new StringBuffer();
- if (!state.isEmpty()) {
- buf.append(prefix.length() == 0 ? "." : prefix).append("?");
- String[] arr = state.keySet().toArray(new String[state.keySet().size()]);
- for (int i = 0; i < arr.length; ++i) {
- buf.append(arr[i]).append("=").append(URLEncoder.encode(state.get(arr[i]), "UTF-8"));
- if (i < arr.length - 1) {
- buf.append("&");
- }
- }
- buf.append(" ");
- }
- if (prefix.length() > 0) {
- prefix += "/";
- }
- String[] keys = children.keySet().toArray(new String[children.keySet().size()]);
- Arrays.sort(keys);
- for (String loc : keys) {
- buf.append(children.get(loc).toString(prefix + URLEncoder.encode(loc, "UTF-8")));
- }
- return buf.toString();
- }
-
- @Override
- public String toString() {
- try {
- return toString("").trim();
- }
- catch (UnsupportedEncodingException e) {
- return e.toString();
- }
- }
-}
+package com.yahoo.documentapi.messagebus.systemstate.rule; + +import com.yahoo.log.LogLevel; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class NodeState { + + /** A location string that expresses the use of the PARENT node. */ + public static final String NODE_PARENT = ".."; + + /** A location string that expresses the use of THIS node. */ + public static final String NODE_CURRENT = "."; + + private static Logger log = Logger.getLogger(NodeState.class.getName()); + private final Map<String, NodeState> children = new LinkedHashMap<String, NodeState>(); + private final Map<String, String> state = new LinkedHashMap<String, String>(); + private NodeState parent = null; + private String id = null; + + /** + * Creates a node state that no internal content. + */ + public NodeState() { + // empty + } + + /** + * Creates a node state based on a list of argument objects. These arguments are iterated and added to this node's + * internal state map. + * + * @param args The arguments to use as state. + */ + public NodeState(List<Argument> args) { + for (Argument arg : args) { + setState(arg.getName(), arg.getValue()); + } + } + + /** + * Adds a child to this node at the given location. The key can be a location string, in which case the necessary + * intermediate node states are created. + * + * @param key The location at which to add the child. + * @param child The child node to add. + * @return This, to allow chaining. + */ + public NodeState addChild(String key, NodeState child) { + getChild(key, true).copy(child); + return this; + } + + /** + * Returns the child at the given location relative to this. + * + * @param key The location of the child to return. + * @return The child object, null if not found. + */ + public NodeState getChild(String key) { + return getChild(key, false); + } + + /** + * Returns the child at the given location relative to this. This method can be forced to return a child node even + * if it does not exist, by adding all intermediate nodes and the target node itself. + * + * @param key The location of the child to return. + * @param force Whether or not to force a return value by creating missing nodes. + * @return The child object, null if not found. + */ + public NodeState getChild(String key, boolean force) { + if (key == null || key.length() == 0) { + return this; + } + String arr[] = key.split("/", 2); + while (arr.length == 2 && arr[0].equals(NODE_CURRENT)) { + arr = arr[1].split("/", 2); + } + if (arr[0].equals(NODE_CURRENT)) { + return this; + } + if (arr[0].equals(NODE_PARENT)) { + if (parent == null) { + log.log(LogLevel.ERROR, "Location string '" + key + "' requests a parent above the top-most node, " + + "returning self to avoid crash."); + } + return parent.getChild(arr[1], force); + } + if (!children.containsKey(arr[0])) { + if (!force) { + return null; + } + children.put(arr[0], new NodeState()); + children.get(arr[0]).setParent(this, arr[0]); + } + if (arr.length == 2) { + return children.get(arr[0]).getChild(arr[1], force); + } + return children.get(arr[0]); + } + + /** + * Returns the map of child nodes for iteration. + * + * @return The internal child map. + */ + public Map<String, NodeState> getChildren() { + return children; + } + + /** + * Removes the named child node from this node, and attempts to compact the system state from this node upwards by + * removing empty nodes. + * + * @param key The child to remove. + * @return The result of invoking {@link #compact} after the remove. + */ + public NodeState removeChild(String key) { + if (key == null || key.length() == 0) { + return this; + } + int pos = key.lastIndexOf("/"); + if (pos > -1) { + NodeState parent = getChild(key.substring(0, pos), false); + if (parent != null) { + return parent.removeChild(key.substring(pos + 1)); + } + } + else { + children.remove(key); + } + return compact(); + } + + /** + * Retrieves some arbitrary state information for a given key. The key can be a location string, in which case the + * necessary intermediate nodes are traversed. If the key is not found, this method returns null. + * + * @param key The name of the state information to return. + * @return The value of the state key. + */ + public String getState(String key) { + if (key == null || key.length() == 0) { + return null; + } + int pos = key.lastIndexOf("/"); + if (pos > -1) { + NodeState parent = getChild(key.substring(0, pos), false); + return parent != null ? parent.getState(key.substring(pos + 1)) : null; + } + return state.get(key); + } + + /** + * Sets some arbitrary state data in this node. The key can be a location string, in which case the necessary + * intermediate nodes are traversed and even created if missing. + * + * @param key The key to set. + * @param value The value to assign to the key. + * @return This, to allow chaining. + */ + public NodeState setState(String key, String value) { + if (key == null || key.length() == 0) { + return this; + } + int pos = key.lastIndexOf("/"); + if (pos > -1) { + getChild(key.substring(0, pos), true).setState(key.substring(pos + 1), value); + } + else { + if (value == null || value.length() == 0) { + return removeState(key); + } + else { + state.put(key, value); + } + } + return this; + } + + /** + * Removes the named (key, value) state pair from this node, and attempts to compact the system state from this node + * upwards by removing empty nodes. + * + * @param key The state variable to clear. + * @return The result of invoking {@link #compact} after the remove. + */ + public NodeState removeState(String key) { + if (key == null || key.length() == 0) { + return this; + } + int pos = key.lastIndexOf("/"); + if (pos > -1) { + NodeState parent = getChild(key.substring(0, pos), false); + if (parent != null) { + return parent.removeState(key.substring(pos + 1)); + } + } + else { + state.remove(key); + } + return compact(); + } + + /** + * Compacts the system state tree from this node upwards. This will delete itself if it has a parent, but no + * internal state and no children. + * + * @return This or the first non-null ancestor, to allow chaining. + */ + private NodeState compact() { + if (state.isEmpty() && children.isEmpty()) { + if (parent != null) { + return parent.removeChild(id); + } + } + return this; + } + + /** + * Copies the state content of another node state object into this. + * + * @param node The node state to copy into this. + * @return This, to allow chaining. + */ + public NodeState copy(NodeState node) { + for (String key : node.state.keySet()) { + state.put(key, node.state.get(key)); + } + for (String key : node.children.keySet()) { + getChild(key, true).copy(node.children.get(key)); + } + return this; + } + + /** + * Clears both the internal state and child list, then compacts the tree from this node upwards. + * + * @return The result of invoking {@link #compact} after the remove. + */ + public NodeState clear() { + state.clear(); + children.clear(); + return compact(); + } + + /** + * Sets the parent of this node. + * + * @param parent The parent node. + * @param id The identifier of this node as seen in the parent. + * @return This, to allow chaining. + */ + public NodeState setParent(NodeState parent, String id) { + this.parent = parent; + this.id = id; + return this; + } + + /** + * Returns a string representation of this node state. + * + * @param prefix The prefix to use for this string. + * @return A string representation of this. + * @throws UnsupportedEncodingException Thrown if the host system does not support UTF-8 encoding. + */ + private String toString(String prefix) throws UnsupportedEncodingException { + StringBuffer buf = new StringBuffer(); + if (!state.isEmpty()) { + buf.append(prefix.length() == 0 ? "." : prefix).append("?"); + String[] arr = state.keySet().toArray(new String[state.keySet().size()]); + for (int i = 0; i < arr.length; ++i) { + buf.append(arr[i]).append("=").append(URLEncoder.encode(state.get(arr[i]), "UTF-8")); + if (i < arr.length - 1) { + buf.append("&"); + } + } + buf.append(" "); + } + if (prefix.length() > 0) { + prefix += "/"; + } + String[] keys = children.keySet().toArray(new String[children.keySet().size()]); + Arrays.sort(keys); + for (String loc : keys) { + buf.append(children.get(loc).toString(prefix + URLEncoder.encode(loc, "UTF-8"))); + } + return buf.toString(); + } + + @Override + public String toString() { + try { + return toString("").trim(); + } + catch (UnsupportedEncodingException e) { + return e.toString(); + } + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java index 99094b09a43..bd7e8804133 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java @@ -1,1540 +1,1540 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi;
-
-import com.yahoo.document.select.parser.ParseException;
-import com.yahoo.documentapi.ProgressToken;
-import com.yahoo.documentapi.VisitorIterator;
-import junit.framework.TestCase;
-import com.yahoo.document.BucketId;
-import com.yahoo.document.BucketIdFactory;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.Vector;
-
-/**
- * Tests for VisitorIterator and ProgressToken (kept in one test case because their
- * interactions are so tightly coupled)
- * @author <a href="mailto:vekterli@yahoo-inc.com">Tor Brede Vekterli</a>
- */
-public class VisitorIteratorTestCase extends TestCase {
-
- public void testIterationSingleBucketUpdate() throws ParseException {
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken progress = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.user = 1234", idFactory, 1, progress);
-
- assertFalse(progress.hasActive());
- assertEquals(progress.getPendingBucketCount(), 1);
- assertEquals(progress.getFinishedBucketCount(), 0);
- assertEquals(progress.getTotalBucketCount(), 1);
- assertFalse(iter.isDone());
- assertTrue(iter.hasNext());
- assertEquals(iter.getRemainingBucketCount(), 1);
- VisitorIterator.BucketProgress b1 = iter.getNext();
- // Upon first getNext of a superbucket, progress == 0
- assertEquals(b1.getSuperbucket(), new BucketId(32, 1234));
- assertEquals(b1.getProgress(), new BucketId());
- assertFalse(iter.hasNext());
- assertFalse(iter.isDone());
- assertEquals(iter.getRemainingBucketCount(), 1);
- // Should only be one active bucket; the one we just got
- assertEquals(progress.getActiveBucketCount(), 1);
- // No pending yet
- assertFalse(progress.hasPending());
- // Update the bucket with a sub-bucket, moving it from active to pending
- BucketId sub = new BucketId(b1.getSuperbucket().getUsedBits() + 1, b1.getSuperbucket().getId());
- iter.update(b1.getSuperbucket(), sub);
- assertFalse(progress.hasActive());
- assertEquals(progress.getPendingBucketCount(), 1);
- assertTrue(iter.hasNext());
- assertFalse(iter.isDone());
- assertEquals(iter.getRemainingBucketCount(), 1);
- // Get the pending bucket
- VisitorIterator.BucketProgress b2 = iter.getNext();
- assertEquals(b2.getSuperbucket(), new BucketId(32, 1234));
- assertEquals(b2.getProgress(), new BucketId(33, 1234));
- assertFalse(iter.hasNext());
- assertEquals(progress.getActiveBucketCount(), 1);
- assertFalse(progress.hasPending());
- // Now update with progress==super, signalling that the bucket is done
- iter.update(b1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- assertFalse(progress.hasActive());
- assertFalse(progress.hasPending());
- assertFalse(iter.hasNext());
- assertTrue(iter.isDone());
- assertTrue(progress.isFinished());
- assertEquals(progress.getFinishedBucketCount(), 1);
- assertEquals(iter.getRemainingBucketCount(), 0);
- }
-
- public void testProgressSerializationRange() throws ParseException {
- int distBits = 4;
-
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken progress = new ProgressToken();
-
- // docsel will be unknown --> entire bucket range will be covered
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, distBits, progress);
-
- assertEquals(progress.getDistributionBitCount(), distBits);
- assertTrue(iter.getBucketSource() instanceof VisitorIterator.DistributionRangeBucketSource);
-
- assertEquals(progress.getFinishedBucketCount(), 0);
- assertEquals(progress.getTotalBucketCount(), 1 << distBits);
-
- // First, get+update half of the buckets, marking them as done
- long bucketCount = 0;
- long bucketStop = 1 << (distBits - 1);
-
- while (iter.hasNext() && bucketCount != bucketStop) {
- VisitorIterator.BucketProgress ids = iter.getNext();
- iter.update(ids.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- ++bucketCount;
- }
- assertEquals(bucketCount, bucketStop);
- // Should be no buckets in limbo at this point
- assertFalse(progress.hasActive());
- assertFalse(progress.hasPending());
- assertFalse(iter.isDone());
- assertTrue(iter.hasNext());
- assertEquals(progress.getFinishedBucketCount(), bucketCount);
- assertFalse(progress.isFinished());
-
- StringBuilder desired = new StringBuilder();
- desired.append("VDS bucket progress file (50.0% completed)\n");
- desired.append(distBits);
- desired.append('\n');
- desired.append(bucketCount); // Finished == cursor for this
- desired.append('\n');
- desired.append(bucketCount);
- desired.append('\n');
- desired.append(1 << distBits);
- desired.append('\n');
-
- assertEquals(desired.toString(), progress.toString());
-
- // Test import, in which case distribution bits are 1
- BucketIdFactory idFactory2 = new BucketIdFactory();
-
- // De-serialization with no pending buckets
- {
- ProgressToken progDs = new ProgressToken(progress.toString());
-
- assertEquals(progDs.getDistributionBitCount(), distBits);
- assertEquals(progDs.getTotalBucketCount(), 1 << distBits);
- assertEquals(progDs.getFinishedBucketCount(), bucketCount);
-
- VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory2, 1, progDs);
-
- assertFalse(progDs.hasPending());
- assertFalse(progDs.hasActive());
- assertTrue(iterDs.hasNext());
- assertFalse(iterDs.isDone());
- assertEquals(distBits, iterDs.getDistributionBitCount());
- assertEquals(distBits, progDs.getDistributionBitCount());
-
- // Iterator must start up on next bucket in range
- VisitorIterator.BucketProgress idDs = iterDs.getNext();
- long resumeKey = ProgressToken.makeNthBucketKey(bucketCount, distBits);
- assertEquals(idDs.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(resumeKey)));
- assertEquals(idDs.getProgress(), new BucketId());
- }
-
- // Now fetch a subset of the remaining buckets without finishing them,
- // keeping some in the active set and some in pending
- int pendingTotal = 1 << (distBits - 3);
- int activeTotal = 1 << (distBits - 3);
- Vector<VisitorIterator.BucketProgress> buckets = new Vector<VisitorIterator.BucketProgress>();
-
- // Pre-fetch, since otherwise we'd reuse pending buckets
- for (int i = 0; i < pendingTotal + activeTotal; ++i) {
- buckets.add(iter.getNext());
- }
-
- for (int i = 0; i < pendingTotal + activeTotal; ++i) {
- VisitorIterator.BucketProgress idTemp = buckets.get(i);
- if (i < activeTotal) {
- // Make them 50% done
- iter.update(idTemp.getSuperbucket(),
- new BucketId(distBits + 2, idTemp.getSuperbucket().getId() | (2 << distBits)));
- }
- // else: leave hanging as active
- }
-
- assertEquals(progress.getActiveBucketCount(), activeTotal);
- assertEquals(progress.getPendingBucketCount(), pendingTotal);
-
- // we can't reuse the existing string builder, since the bucket cursor
- // has changed
- desired = new StringBuilder();
- desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
- desired.append(distBits);
- desired.append('\n');
- desired.append(bucketCount + pendingTotal + activeTotal);
- desired.append('\n');
- desired.append(bucketCount);
- desired.append('\n');
- desired.append(1 << distBits);
- desired.append('\n');
-
- assertEquals(progress.getBuckets().entrySet().size(), pendingTotal + activeTotal);
-
- for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
- : progress.getBuckets().entrySet()) {
- desired.append(Long.toHexString(ProgressToken.keyToBucketId(entry.getKey().getKey())));
- desired.append(':');
- desired.append(Long.toHexString(entry.getValue().getProgress().getRawId()));
- desired.append('\n');
- }
-
- assertEquals(progress.toString(), desired.toString());
-
- {
- // Deserialization with pending buckets
- ProgressToken progDs = new ProgressToken(progress.toString());
-
- assertEquals(progDs.getDistributionBitCount(), distBits);
- assertEquals(progDs.getTotalBucketCount(), 1 << distBits);
- assertEquals(progDs.getFinishedBucketCount(), bucketCount);
-
- VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory2, 1, progDs);
-
- // All started but nonfinished buckets get placed in pending upon
- // deserialization
- assertEquals(progDs.getPendingBucketCount(), pendingTotal + activeTotal);
- assertEquals(distBits, progDs.getDistributionBitCount());
- assertEquals(distBits, iterDs.getDistributionBitCount());
- assertFalse(progDs.hasActive());
- assertTrue(iterDs.hasNext());
- assertFalse(iterDs.isDone());
- assertEquals(progDs.getBucketCursor(), bucketCount + pendingTotal + activeTotal);
- }
-
- // Finish all the active buckets
- for (int i = activeTotal; i < activeTotal + pendingTotal; ++i) {
- iter.update(buckets.get(i).getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- ++bucketCount;
- }
-
- assertEquals(progress.getActiveBucketCount(), 0);
- boolean consistentNext = true;
- // Get all pending/remaining sourced and finish them all
- while (!iter.isDone()) {
- if (!iter.hasNext()) {
- consistentNext = false;
- break;
- }
- VisitorIterator.BucketProgress bp = iter.getNext();
- iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- ++bucketCount;
- }
-
- assertTrue(consistentNext);
- assertFalse(iter.hasNext());
- assertTrue(progress.isFinished());
- // Cumulative number of finished buckets must match 2^distbits
- assertEquals(bucketCount, 1 << distBits);
- StringBuilder finished = new StringBuilder();
- finished.append("VDS bucket progress file (100.0% completed)\n");
- finished.append(distBits);
- finished.append('\n');
- finished.append(1 << distBits); // Cursor
- finished.append('\n');
- finished.append(1 << distBits); // Finished
- finished.append('\n');
- finished.append(1 << distBits); // Total
- finished.append('\n');
-
- assertEquals(progress.toString(), finished.toString());
- }
-
- public void testProgressSerializationExplicit() throws ParseException {
- int distBits = 16;
-
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken progress = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, progress);
-
- assertEquals(progress.getDistributionBitCount(), distBits);
- assertTrue(iter.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
-
- assertEquals(progress.getFinishedBucketCount(), 0);
- assertEquals(progress.getTotalBucketCount(), 3);
- assertEquals(progress.getPendingBucketCount(), 3);
-
- VisitorIterator.BucketProgress bp1 = iter.getNext();
- VisitorIterator.BucketProgress bp2 = iter.getNext();
- assertEquals(progress.getPendingBucketCount(), 1);
- assertEquals(progress.getActiveBucketCount(), 2);
- // Buckets are ordered by their reverse bucket id key
- assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
- assertEquals(bp1.getProgress(), new BucketId());
- // Put bucket 1234 back into pending
- iter.update(bp1.getSuperbucket(), new BucketId(36, 1234));
- assertEquals(progress.getPendingBucketCount(), 2);
-
- assertEquals(bp2.getSuperbucket(), new BucketId(32, 8009));
- assertEquals(bp2.getProgress(), new BucketId());
-
- {
- StringBuilder desired = new StringBuilder();
- desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
- desired.append(distBits);
- desired.append('\n');
- desired.append(0);
- desired.append('\n');
- desired.append(0);
- desired.append('\n');
- desired.append(3);
- desired.append('\n');
- // Pending/active buckets are written in an increasing (key, not
- // bucket-id!) order
- desired.append(Long.toHexString(new BucketId(32, 1234).getRawId()));
- desired.append(':');
- desired.append(Long.toHexString(new BucketId(36, 1234).getRawId()));
- desired.append('\n');
- desired.append(Long.toHexString(new BucketId(32, 8009).getRawId()));
- desired.append(":0\n");
- desired.append(Long.toHexString(new BucketId(32, 6789).getRawId()));
- desired.append(":0\n");
-
- assertEquals(desired.toString(), progress.toString());
-
- ProgressToken prog2 = new ProgressToken(progress.toString());
- assertEquals(prog2.getDistributionBitCount(), distBits);
- assertEquals(prog2.getTotalBucketCount(), 3);
- assertEquals(prog2.getFinishedBucketCount(), 0);
-
- VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
- "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, prog2);
-
- assertEquals(prog2.getPendingBucketCount(), 3);
- assertFalse(prog2.hasActive());
- assertTrue(iter2.hasNext());
- assertFalse(iter2.isDone());
-
- assertTrue(iter2.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
- assertFalse(iter2.getBucketSource().hasNext());
-
- VisitorIterator.BucketProgress bp = iter2.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(32, 1234));
- assertEquals(bp.getProgress(), new BucketId(36, 1234));
- assertEquals(prog2.getPendingBucketCount(), 2);
-
- assertTrue(iter2.hasNext());
- assertFalse(iter2.isDone());
- bp = iter2.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(32, 8009));
- assertEquals(bp.getProgress(), new BucketId());
- assertEquals(prog2.getPendingBucketCount(), 1);
-
- assertTrue(iter2.hasNext());
- assertFalse(iter2.isDone());
- bp = iter2.getNext();
- assertEquals(prog2.getPendingBucketCount(), 0);
- assertEquals(bp.getSuperbucket(), new BucketId(32, 6789));
- assertEquals(bp.getProgress(), new BucketId());
- assertFalse(iter2.hasNext());
- assertFalse(iter2.isDone()); // Active buckets
- assertEquals(prog2.getActiveBucketCount(), 3);
- }
-
- // Finish off all active buckets
- assertTrue(iter.hasNext());
- assertFalse(iter.isDone());
- bp1 = iter.getNext();
- assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
- assertEquals(bp1.getProgress(), new BucketId(36, 1234));
-
- iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
-
- assertTrue(iter.hasNext());
- assertFalse(iter.isDone());
- bp1 = iter.getNext();
- assertEquals(bp1.getSuperbucket(), new BucketId(32, 6789));
- assertEquals(bp1.getProgress(), new BucketId());
-
- // Just to make sure Java serializes the long properly
- assertEquals(
- progress.toString(),
- "VDS bucket progress file (" + progress.percentFinished() + "% completed)\n" +
- "16\n" +
- "0\n" +
- "1\n" +
- "3\n" +
- "8000000000001f49:0\n" +
- "8000000000001a85:0\n");
-
- iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
-
- // At this point, we've got one active but no pending, so hasNext == false,
- // but isDone is also == false
- assertFalse(iter.hasNext());
- assertFalse(iter.isDone());
- assertEquals(progress.getPendingBucketCount(), 0);
- assertEquals(progress.getActiveBucketCount(), 1);
-
- iter.update(bp2.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- assertFalse(iter.hasNext());
- assertTrue(iter.isDone());
- assertTrue(progress.isFinished());
- assertEquals(progress.getActiveBucketCount(), 0);
-
- {
- StringBuilder finished = new StringBuilder();
- finished.append("VDS bucket progress file (100.0% completed)\n");
- finished.append(distBits);
- finished.append('\n');
- finished.append(0); // Cursor (not used by explicit)
- finished.append('\n');
- finished.append(3); // Finished
- finished.append('\n');
- finished.append(3); // Total
- finished.append('\n');
-
- assertEquals(finished.toString(), progress.toString());
- }
- }
-
- /**
- * Test that doing update() on a bucket several times in a row (without re-fetching
- * from getNext first) works
- * @throws ParseException
- */
- public void testActiveUpdate() throws ParseException {
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken progress = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group = \"yahoo.com\"", idFactory, 16, progress);
-
- VisitorIterator.BucketProgress bp = iter.getNext();
-
- assertEquals(progress.getPendingBucketCount(), 0);
- assertEquals(progress.getActiveBucketCount(), 1);
-
- BucketId superbucket = bp.getSuperbucket();
- int usedBits = superbucket.getUsedBits();
-
- iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (2L << usedBits)));
- assertEquals(progress.getPendingBucketCount(), 1);
- assertEquals(progress.getActiveBucketCount(), 0);
- iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
- assertEquals(progress.getPendingBucketCount(), 1);
- assertEquals(progress.getActiveBucketCount(), 0);
-
- bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), superbucket);
- assertEquals(bp.getProgress(), new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits)));
- assertEquals(progress.getPendingBucketCount(), 0);
- assertEquals(progress.getActiveBucketCount(), 1);
- }
-
- /**
- * Test that ensures doing update(superbucket, 0) simply puts the bucket back in
- * pending
- * @throws ParseException
- */
- public void testNullAndSuperUpdate() throws ParseException {
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken progress = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group = \"yahoo.com\"", idFactory, 16, progress);
-
- assertEquals(progress.getPendingBucketCount(), 1);
-
- VisitorIterator.BucketProgress bp = iter.getNext();
- assertEquals(bp.getProgress(), new BucketId());
- BucketId superbucket = bp.getSuperbucket();
- BucketId sub = bp.getProgress();
-
- assertFalse(iter.hasNext());
- assertFalse(iter.isDone());
- assertEquals(progress.getPendingBucketCount(), 0);
- assertEquals(progress.getActiveBucketCount(), 1);
-
- // 0-bucket
- iter.update(superbucket, ProgressToken.NULL_BUCKET);
- assertTrue(iter.hasNext());
- assertFalse(iter.isDone());
- assertEquals(progress.getPendingBucketCount(), 1);
- assertEquals(progress.getActiveBucketCount(), 0);
-
- VisitorIterator.BucketProgress bp2 = iter.getNext();
- assertEquals(bp2.getSuperbucket(), superbucket);
- assertEquals(bp2.getProgress(), ProgressToken.NULL_BUCKET);
- assertEquals(progress.getPendingBucketCount(), 0);
- assertEquals(progress.getActiveBucketCount(), 1);
-
- // progress == super
- iter.update(superbucket, superbucket);
- assertTrue(iter.hasNext());
- assertFalse(iter.isDone());
- assertEquals(progress.getPendingBucketCount(), 1);
- assertEquals(progress.getActiveBucketCount(), 0);
-
- bp2 = iter.getNext();
- assertEquals(bp2.getSuperbucket(), superbucket);
- assertEquals(bp2.getProgress(), superbucket);
- assertEquals(progress.getPendingBucketCount(), 0);
- assertEquals(progress.getActiveBucketCount(), 1);
- }
-
- public void testDeserializedFinishedProgress() {
- StringBuilder finished = new StringBuilder();
- finished.append("VDS bucket progress file\n"); // legacy; no completion percentage
- finished.append(17);
- finished.append('\n');
- finished.append(1L << 17); // Cursor
- finished.append('\n');
- finished.append(1L << 17); // Finished
- finished.append('\n');
- finished.append(1L << 17); // Total
- finished.append('\n');
-
- ProgressToken token = new ProgressToken(finished.toString());
- assertEquals(token.getDistributionBitCount(), 17);
- assertEquals(token.getTotalBucketCount(), 1L << 17);
- assertEquals(token.getFinishedBucketCount(), 1L << 17);
- assertEquals(token.getBucketCursor(), 1L << 17);
- assertTrue(token.isFinished());
-
- ProgressToken token2 = new ProgressToken(token.serialize());
- assertEquals(17, token2.getDistributionBitCount());
- assertEquals(1L << 17, token2.getTotalBucketCount());
- assertEquals(1L << 17, token2.getFinishedBucketCount());
- assertEquals(1L << 17, token2.getBucketCursor());
- assertTrue(token2.isFinished());
- }
-
- public void testBucketProgressFraction() {
- double epsilon = 0.00001;
- // No progress
- BucketId b_0 = new BucketId();
- // No split; only superbucket (100%)
- BucketId b_100_0 = new BucketId(16, 1234);
- // 1 split (1/2)
- BucketId b_50_1 = new BucketId(17, 1234);
- BucketId b_100_1 = new BucketId(17, 1234 | (1 << 16));
- // 2 splits (1/4)
- BucketId b_25_2 = new BucketId(18, 1234);
- BucketId b_50_2 = new BucketId(18, 1234 | (2 << 16));
- BucketId b_75_2 = new BucketId(18, 1234 | (1 << 16));
- BucketId b_100_2 = new BucketId(18, 1234 | (3 << 16));
-
- ProgressToken p = new ProgressToken(16);
-
- BucketId sb = new BucketId(16, 1234);
-
- assertEquals(p.progressFraction(new BucketId(32, 1234), b_0), 0.0, epsilon);
-
- assertEquals(p.progressFraction(sb, b_100_0), 1.0, epsilon);
-
- assertEquals(p.progressFraction(sb, b_50_1), 0.5, epsilon);
- assertEquals(p.progressFraction(sb, b_100_1), 1.0, epsilon);
-
- assertEquals(p.progressFraction(sb, b_25_2), 0.25, epsilon);
- assertEquals(p.progressFraction(sb, b_50_2), 0.5, epsilon);
- assertEquals(p.progressFraction(sb, b_75_2), 0.75, epsilon);
- assertEquals(p.progressFraction(sb, b_100_2), 1.0, epsilon);
-
- assertEquals(p.progressFraction(new BucketId(0x8000000000000000L),
- new BucketId(0xb0000fff00000000L)), 1.0, epsilon);
- }
-
- public void testProgressEstimation() throws ParseException {
- int distBits = 4;
-
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken progress = new ProgressToken();
-
- // Create a range of [0, 16) superbuckets
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, distBits, progress);
-
- assertEquals(progress.getDistributionBitCount(), 4);
-
- double epsilon = 0.00001;
- assertEquals(progress.percentFinished(), 0, epsilon);
- VisitorIterator.BucketProgress bp = iter.getNext();
- // Finish first superbucket (6.25% total)
- iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- assertEquals(progress.percentFinished(), 6.25, epsilon);
- assertEquals(progress.getFinishedBucketCount(), 1);
-
- bp = iter.getNext();
- VisitorIterator.BucketProgress bp3 = iter.getNext();
- VisitorIterator.BucketProgress bp4 = iter.getNext();
-
- // Finish second (12.5% total)
- iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- assertEquals(progress.percentFinished(), 12.5, epsilon);
- assertEquals(progress.getFinishedBucketCount(), 2);
-
- // Finish third bucket 75% through (17.1875% total)
- iter.update(bp3.getSuperbucket(), new BucketId(distBits + 2, bp3.getSuperbucket().getId() | (1 << distBits)));
- assertEquals(progress.percentFinished(), 17.1875, epsilon);
- assertEquals(progress.getFinishedBucketCount(), 2);
-
- // Finish fourth bucket 25% through (18.75% total)
- iter.update(bp4.getSuperbucket(), new BucketId(distBits + 2, bp4.getSuperbucket().getId()));
- assertEquals(progress.percentFinished(), 18.75, epsilon);
- assertEquals(progress.getFinishedBucketCount(), 2);
- // Finish all buckets
- iter.update(bp4.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- iter.update(bp3.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- assertEquals(progress.percentFinished(), 25, epsilon);
- assertEquals(progress.getFinishedBucketCount(), 4);
-
- while (iter.hasNext()) {
- bp = iter.getNext();
- iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- }
-
- assertEquals(progress.getFinishedBucketCount(), 16);
- assertEquals(progress.percentFinished(), 100, epsilon);
- }
-
- public void testBucketKeyWrapperOrdering() {
- ProgressToken.BucketKeyWrapper bk1 = new ProgressToken.BucketKeyWrapper(0x0000000000000001L);
- ProgressToken.BucketKeyWrapper bk2 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL);
- ProgressToken.BucketKeyWrapper bk3 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L);
- ProgressToken.BucketKeyWrapper bk4 = new ProgressToken.BucketKeyWrapper(0xFFFFFFFFFFFFFFFFL);
- assertTrue(bk1.compareTo(bk2) < 0);
- assertTrue(bk2.compareTo(bk3) < 0);
- assertTrue(bk3.compareTo(bk4) < 0);
- assertTrue(bk2.compareTo(bk1) > 0);
- assertTrue(bk3.compareTo(bk2) > 0);
- assertTrue(bk4.compareTo(bk3) > 0);
- ProgressToken.BucketKeyWrapper bk5 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL);
- ProgressToken.BucketKeyWrapper bk6 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L);
- assertTrue(bk5.compareTo(bk2) == 0);
- assertTrue(bk6.compareTo(bk3) == 0);
- }
-
- private void doTestBucketKeyGeneration(int db) {
- // Can't use longs since they won't sort properly when MSB is set
- ProgressToken.BucketKeyWrapper[] keys = new ProgressToken.BucketKeyWrapper[1 << db];
-
- // Generate entire bucket space for db
- for (int i = 0; i < (1 << db); ++i) {
- keys[i] = new ProgressToken.BucketKeyWrapper(
- ProgressToken.bucketToKey(new BucketId(db, i).getId()));
- }
- Arrays.sort(keys);
-
- boolean consistentKeys = true;
- // Verify that makeNthBucketKey yields the same result as the equivalent
- // ordered value in the array of keys
- for (int i = 0; i < (1 << db); ++i) {
- long genKey = ProgressToken.makeNthBucketKey(i, db);
- long knownKey = keys[i].getKey();
- if (genKey != knownKey) {
- consistentKeys = false;
- break;
- }
- }
- assertTrue(consistentKeys);
- }
-
- public void testBucketKeyGeneration() {
- // Due to the number of objects needed to be allocated, only test for a
- // small set of distribution bits
- for (int i = 1; i < 14; ++i) {
- doTestBucketKeyGeneration(i);
- }
- }
-
- public void testSingleBucketSplits() throws ParseException {
- int db = 2;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- // Create a range of [0, 4) superbuckets
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
- VisitorIterator.BucketProgress bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
- // Put back as pending
- iter.update(bp.getSuperbucket(), new BucketId());
- assertEquals(p.getPendingBucketCount(), 1);
- p.splitPendingBucket(new BucketId(db, 0));
- assertEquals(p.getPendingBucketCount(), 2);
- bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 0)); // left split
- assertEquals(bp.getProgress(), new BucketId(0));
- bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 4)); // right split
- assertEquals(bp.getProgress(), new BucketId(0));
-
- bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db, 2));
- // Put back as pending, with a progress of 10010. This implies splitting
- // the bucket should set both splits with a progress to 10010
- iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x12));
- assertEquals(p.getPendingBucketCount(), 1);
- p.splitPendingBucket(new BucketId(db, 2));
- assertEquals(p.getPendingBucketCount(), 2);
- bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 2)); // left split
- assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
- bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 6)); // right split
- assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12));
-
- bp = iter.getNext();
- // Put back as pending with a progress of 10101. This implies splitting the
- // bucket should _discard_ left and set right's progress to 10101.
- // Update: no it shouldn't, we now split with equal progress without
- // discarding
- assertEquals(bp.getSuperbucket(), new BucketId(db, 1));
- iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x15));
- assertEquals(p.getPendingBucketCount(), 1);
- p.splitPendingBucket(new BucketId(db, 1));
- assertEquals(p.getPendingBucketCount(), 2);
- bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 1));
- assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
- bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 5)); // right split
- assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15));
- }
-
- /**
- * Test increasing the distribution bits for a full bucket space range
- * source with no finished, active or pending buckets
- * @throws ParseException upon docsel parse failure (shouldn't happen)
- */
- public void testRangeDistributionBitIncrease1NoPending() throws ParseException {
- int db = 2;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- // Test for empty progress token. no splitting involved
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- assertEquals(p.getTotalBucketCount(), 4);
- iter.setDistributionBitCount(db + 1);
- assertEquals(p.getTotalBucketCount(), 8);
- assertEquals(p.getDistributionBitCount(), db + 1);
- assertEquals(iter.getDistributionBitCount(), db + 1);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
-
- int[] desired = new int[] { 0, 4, 2, 6, 1, 5, 3, 7 };
- for (int i = 0; i < 8; ++i) {
- VisitorIterator.BucketProgress bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db + 1, desired[i]));
- }
- }
-
- public void testRangeDistributionBitIncrease1AllBucketStates() throws ParseException {
- int db = 3;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- // For this test, have 1 finished bucket, 3 pending and 0 active (we
- // want to have the splitting to be triggered immediately)
- VisitorIterator.BucketProgress bp = iter.getNext();
- iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
- bpp[0] = iter.getNext();
- bpp[1] = iter.getNext();
- bpp[2] = iter.getNext();
- iter.update(bpp[0].getSuperbucket(), new BucketId());
- iter.update(bpp[1].getSuperbucket(), new BucketId());
- iter.update(bpp[2].getSuperbucket(), new BucketId());
-
- assertEquals(p.getFinishedBucketCount(), 1);
- assertEquals(p.getPendingBucketCount(), 3);
- assertEquals(p.getActiveBucketCount(), 0);
-
- iter.setDistributionBitCount(db + 1);
-
- assertEquals(p.getTotalBucketCount(), 16);
- assertEquals(p.getFinishedBucketCount(), 2);
- assertEquals(p.getPendingBucketCount(), 6);
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(p.getDistributionBitCount(), db + 1);
- assertEquals(iter.getDistributionBitCount(), db + 1);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1);
-
- // Bucket 3:0x4 -> 4:0x4 & 4:0xC
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x04));
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0C));
- // Bucket 3:0x2 -> 4:0x2 & 4:0xA
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x02));
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0A));
- // Bucket 3:0x6 -> 4:0x6 & 4:0xE
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x06));
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0E));
-
- assertEquals(p.getPendingBucketCount(), 0);
- // Bucket source should now begin returning from bucket 4:0x1
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x01));
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x09));
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x05));
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0D));
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x03));
- // Assume correct from here on
- }
-
- public void testRangeDistributionIncreaseMultipleBits() throws ParseException {
- int db = 16;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- // For this test, have 3 finished bucket, 2 pending and 1 active
- for (int i = 0; i < 3; ++i) {
- iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- }
-
- VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
- bpp[0] = iter.getNext();
- bpp[1] = iter.getNext();
- VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active
- iter.update(bpp[0].getSuperbucket(), new BucketId());
- iter.update(bpp[1].getSuperbucket(), new BucketId());
-
- iter.setDistributionBitCount(20);
- // ProgressToken doesn't change yet, since it had active buckets
- assertEquals(p.getDistributionBitCount(), 16);
- assertEquals(iter.getDistributionBitCount(), 20);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), 20);
-
- assertFalse(iter.hasNext());
- assertFalse(iter.isDone());
- assertTrue(iter.getBucketSource().shouldYield());
- assertEquals(p.getPendingBucketCount(), 2);
- assertEquals(p.getActiveBucketCount(), 1);
-
- // Finish active, triggering the consistency fixes
- iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
-
- assertEquals(p.getDistributionBitCount(), 20);
- assertEquals(p.getPendingBucketCount(), 32);
- assertEquals(p.getActiveBucketCount(), 0);
- // Each bucket with db:16 becomes equal to 16 buckets with db:20, so
- // the bucket space position must be 16 * 6 = 96
- assertEquals(p.getBucketCursor(), 96);
- // Each finished bucket also covers less ground, so count is upped
- // accordingly
- assertEquals(p.getFinishedBucketCount(), 16 * 4);
-
- // Remove pending that came from the split
- // Bucket space that should be covered by the 32 buckets is [48, 80)
- // when using 20 distribution bits
- for (int i = 0; i < 32; ++i) {
- long testKey = ProgressToken.makeNthBucketKey(i + 48, 20);
- VisitorIterator.BucketProgress bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(testKey)));
- iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- }
- assertEquals(p.getPendingBucketCount(), 0);
- assertEquals(p.getFinishedBucketCount(), 16 * 6);
-
- // Bucket source should now begin returning from bucket 20:0x6000
- assertEquals(iter.getNext().getSuperbucket(), new BucketId(20, 0x6000));
- }
-
- public void testSingleBucketMerge() throws ParseException {
- int db = 2;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- // Create a range of [0, 4) superbuckets
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- VisitorIterator.BucketProgress bp = iter.getNext();
- // Put back as pending and split it
- iter.update(bp.getSuperbucket(), new BucketId());
- p.splitPendingBucket(new BucketId(db, 0));
- assertEquals(p.getPendingBucketCount(), 2);
- // Merge both back into one node. Merge from left sibling with right present
- p.mergePendingBucket(new BucketId(db + 1, 0));
- assertEquals(p.getPendingBucketCount(), 1);
- bp = iter.getNext();
- assertEquals(bp.getSuperbucket(), new BucketId(db, 0));
- }
-
- public void testRangeDistributionBitDecrease1() throws ParseException {
- int db = 16;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- VisitorIterator.DistributionRangeBucketSource src
- = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
-
- assertTrue(src.isLosslessResetPossible());
-
- // For this test, have 3 finished buckets, 6 pending and 1 active
- // This gives a sibling "distribution" of FF FP PP PP PA. When all
- // active buckets have been updated, 3 merges should be triggered
- for (int i = 0; i < 3; ++i) {
- iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- }
-
- assertFalse(src.isLosslessResetPossible());
-
- VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[6];
- for (int i = 0; i < 6; ++i) {
- bpp[i] = iter.getNext();
- }
- VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active
- for (int i = 0; i < 6; ++i) {
- iter.update(bpp[i].getSuperbucket(), new BucketId());
- }
-
- assertEquals(p.getBucketCursor(), 10);
-
- iter.setDistributionBitCount(db - 1);
- assertEquals(iter.getDistributionBitCount(), db - 1);
- assertEquals(p.getDistributionBitCount(), db);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), db - 1);
- // The iterator is waiting patiently for all active buckets to be updated,
- // at which point it will performed the merging and actually updating the
- // progress token's distribution bit count
- assertTrue(iter.getBucketSource().shouldYield());
- assertFalse(iter.hasNext());
- assertFalse(iter.isDone());
- assertEquals(p.getActiveBucketCount(), 1);
- iter.update(bpa.getSuperbucket(), new BucketId());
-
- assertEquals(p.getDistributionBitCount(), db - 1);
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(p.getPendingBucketCount(), 4); // 3 merges, P PP PP PP -> P P P P
-
- assertEquals(p.getFinishedBucketCount(), 1);
- assertEquals(p.getBucketCursor(), 5);
- }
-
- // Test that splitting and merging from and to the same db count gives
- // back the initial state
- public void testRangeDistributionBitIncreaseDecrease() throws ParseException {
- int db = 16;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- VisitorIterator.DistributionRangeBucketSource src
- = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
-
- assertTrue(src.isLosslessResetPossible());
-
- // "Sabotage" resetting by having at least 1 finished
- iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
-
- VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
- for (int i = 0; i < 4; ++i) {
- bpp[i] = iter.getNext();
- }
- for (int i = 0; i < 4; ++i) {
- iter.update(bpp[i].getSuperbucket(), new BucketId());
- }
-
- assertFalse(src.isLosslessResetPossible());
-
- iter.setDistributionBitCount(20);
- assertEquals(p.getDistributionBitCount(), 20);
- assertEquals(p.getPendingBucketCount(), 4 << 4);
- assertFalse(iter.getBucketSource().shouldYield());
- assertEquals(p.getBucketCursor(), 5 << 4);
-
- iter.setDistributionBitCount(16);
-
- assertEquals(p.getDistributionBitCount(), 16);
- assertEquals(p.getPendingBucketCount(), 4);
- assertFalse(iter.getBucketSource().shouldYield());
- assertEquals(p.getBucketCursor(), 5);
- }
-
- // Test that intermittent changes in distribution are handled properly, e.g.
- // changing from 11 -> 9 with X active and then before all those are flushed,
- // the distribution goes up to 12
- public void testRangeDistributionBitChangeWithoutDone() throws ParseException {
- int db = 11;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- VisitorIterator.DistributionRangeBucketSource src
- = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
-
- VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4];
- for (int i = 0; i < 4; ++i) {
- bpp[i] = iter.getNext();
- }
- for (int i = 0; i < 2; ++i) {
- iter.update(bpp[i].getSuperbucket(), new BucketId());
- }
-
- assertFalse(src.isLosslessResetPossible());
-
- // Now 2 pending, 2 active
-
- iter.setDistributionBitCount(9);
- assertEquals(p.getDistributionBitCount(), 11);
- assertEquals(p.getActiveBucketCount(), 2);
- assertEquals(p.getPendingBucketCount(), 2);
- assertTrue(iter.getBucketSource().shouldYield());
- // Update as pending, still with old count since there's 1 more active
- // with bpp[2]. Have progress so that lossless reset isn't possible
- iter.update(bpp[3].getSuperbucket(), new BucketId(15, bpp[3].getSuperbucket().getId()));
-
- iter.setDistributionBitCount(12);
- assertEquals(p.getActiveBucketCount(), 1);
- assertEquals(p.getPendingBucketCount(), 3);
- assertTrue(iter.getBucketSource().shouldYield());
-
- // Serialize before token is updated to 12 bits
- String serialized = p.toString();
-
- iter.update(bpp[2].getSuperbucket(), ProgressToken.FINISHED_BUCKET);
-
- assertEquals(p.getActiveBucketCount(), 0);
- // All active buckets are at db=11, so they should be split once each
- assertEquals(p.getPendingBucketCount(), 3 * 2);
- assertFalse(iter.getBucketSource().shouldYield());
- assertEquals(p.getFinishedBucketCount(), 2);
-
- // Ensure we get a consistent progress token imported
- ProgressToken p2 = new ProgressToken(serialized);
- assertEquals(p2.getDistributionBitCount(), 11); // Not yet updated
-
- BucketIdFactory idFactory2 = new BucketIdFactory();
- VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory2, 1, p2);
-
- // Not yet updated, since we don't trust the initial BucketIdFactory
- assertEquals(iter2.getDistributionBitCount(), 11);
- assertEquals(p2.getDistributionBitCount(), 11);
- iter2.setDistributionBitCount(12);
- // Now it has been updated
- assertEquals(p2.getDistributionBitCount(), 12);
- assertEquals(p2.getPendingBucketCount(), 8);
- assertEquals(p2.getBucketCursor(), 8);
- assertEquals(p2.getFinishedBucketCount(), 0);
- }
-
- // Test a drop from 31->11 bits upon iteration start
- public void testRangeDistributionBitInitialDrop() throws ParseException {
- int db = 31;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[3];
- bp[0] = iter.getNext();
- bp[1] = iter.getNext();
- bp[2] = iter.getNext();
- iter.update(bp[2].getSuperbucket(), new BucketId());
- iter.update(bp[1].getSuperbucket(), new BucketId());
- assertEquals(p.getActiveBucketCount(), 1);
-
- iter.setDistributionBitCount(11);
-
- assertFalse(iter.hasNext());
- assertFalse(iter.isDone());
- assertEquals(p.getActiveBucketCount(), 1);
-
- // Updating the active bucket allows the merging to take place
- iter.update(new BucketId(31, 0), new BucketId());
-
- assertTrue(iter.hasNext());
- assertFalse(iter.isDone());
-
- // All pending buckets should have been merged down to just 1 now
- // Update: now rather gets reset
- assertEquals(p.getPendingBucketCount(), 0);
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(p.getFinishedBucketCount(), 0);
- assertEquals(p.getBucketCursor(), 0);
-
- bp[0] = iter.getNext();
- assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
- }
-
- // Similar to testRangeDistributionBitInitialDrop, but going from 1 to 11
- // This tests that doing so may be done in an optimized way rather than
- // attempting to split enough buckets to cover the entire bucket space!
- public void testRangeDistributionLosslessReset() throws ParseException {
- int db = 1;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- VisitorIterator.DistributionRangeBucketSource src
- = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource();
-
- VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[2];
- bp[0] = iter.getNext();
- bp[1] = iter.getNext();
-
- String serialized = p.toString();
-
- assertFalse(src.isLosslessResetPossible());
-
- iter.update(bp[1].getSuperbucket(), new BucketId());
- assertEquals(p.getActiveBucketCount(), 1);
-
- iter.setDistributionBitCount(11);
-
- assertFalse(src.isLosslessResetPossible());
- assertEquals(p.getDistributionBitCount(), 1); // Still at 1
-
- assertFalse(iter.hasNext());
- assertFalse(iter.isDone());
- assertEquals(p.getActiveBucketCount(), 1);
-
- // Updating the active bucket allows the reset to take place
- iter.update(new BucketId(1, 0), new BucketId());
-
- assertTrue(iter.hasNext());
- assertFalse(iter.isDone());
-
- // Should not be any buckets pending/active and the cursor should be
- // back at 0
- assertEquals(p.getPendingBucketCount(), 0);
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(p.getFinishedBucketCount(), 0);
- assertEquals(p.getBucketCursor(), 0);
- assertEquals(p.getDistributionBitCount(), 11);
-
- bp[0] = iter.getNext();
- assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
-
- // Ensure resetting also works when you're importing existing
- // progress
- p = new ProgressToken(serialized);
- idFactory = new BucketIdFactory();
- iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, 1, p);
-
- iter.setDistributionBitCount(11);
-
- assertEquals(p.getPendingBucketCount(), 0);
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(p.getFinishedBucketCount(), 0);
- assertEquals(p.getBucketCursor(), 0);
- assertEquals(p.getDistributionBitCount(), 11);
-
- bp[0] = iter.getNext();
- assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0));
- }
-
- public void testExplicitDistributionBitIncrease() throws ParseException {
- int distBits = 12;
-
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
-
- assertEquals(iter.getDistributionBitCount(), distBits);
- assertEquals(p.getDistributionBitCount(), distBits);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
-
- iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- iter.setDistributionBitCount(16);
-
- assertEquals(iter.getDistributionBitCount(), 16);
- assertEquals(p.getDistributionBitCount(), 16);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
- // Changing dist bits for explicit source should change nothing
- assertEquals(p.getPendingBucketCount(), 2);
- assertEquals(p.getFinishedBucketCount(), 1);
- assertEquals(p.getTotalBucketCount(), 3);
- }
-
- public void testExplicitDistributionBitDecrease() throws ParseException {
- int distBits = 20;
-
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p);
-
- assertEquals(iter.getDistributionBitCount(), distBits);
- assertEquals(p.getDistributionBitCount(), distBits);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits);
-
- iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- iter.setDistributionBitCount(16);
-
- assertEquals(iter.getDistributionBitCount(), 16);
- assertEquals(p.getDistributionBitCount(), 16);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), 16);
- // Changing dist bits for explicit source should change nothing
- assertEquals(p.getPendingBucketCount(), 2);
- assertEquals(p.getFinishedBucketCount(), 1);
- assertEquals(p.getTotalBucketCount(), 3);
- }
-
- public void testExplicitDistributionImportNoTruncation() throws ParseException {
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, 20, p);
- assertEquals(20, iter.getDistributionBitCount());
- assertEquals(20, p.getDistributionBitCount());
- assertEquals(20, iter.getBucketSource().getDistributionBitCount());
-
- iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
-
- // Make sure no truncation is done on import
- String serialized = p.toString();
- ProgressToken p2 = new ProgressToken(serialized);
- BucketIdFactory idFactory2 = new BucketIdFactory();
- VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
- "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory2, 1, p2);
- assertEquals(20, iter2.getDistributionBitCount());
- assertEquals(20, p2.getDistributionBitCount());
- assertEquals(20, iter2.getBucketSource().getDistributionBitCount());
- assertEquals(2, p2.getPendingBucketCount());
- assertEquals(1, p2.getFinishedBucketCount());
- assertEquals(3, p2.getTotalBucketCount());
- }
-
- public void testImportProgressWithOutdatedDistribution() throws ParseException {
- String input = "VDS bucket progress file\n" +
- "10\n" +
- "503\n" +
- "500\n" +
- "1024\n" +
- "28000000000000be:0\n" +
- "28000000000002be:0\n" +
- "28000000000001be:0\n";
-
- int db = 12;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken(input);
- assertEquals(10, p.getDistributionBitCount());
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, 1, p);
-
- iter.setDistributionBitCount(12);
- assertEquals(iter.getDistributionBitCount(), 12);
- assertEquals(p.getDistributionBitCount(), 12);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), 12);
-
- assertEquals(p.getTotalBucketCount(), 1 << 12);
- assertEquals(p.getFinishedBucketCount(), 500 << 2);
- assertEquals(p.getPendingBucketCount(), 3 << 2);
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(p.getBucketCursor(), 503 << 2);
- assertTrue(iter.hasNext());
-
- ProgressToken p2 = new ProgressToken(p.serialize());
- assertEquals(p2.getDistributionBitCount(), 12);
- assertEquals(p2.getTotalBucketCount(), 1 << 12);
- assertEquals(p2.getFinishedBucketCount(), 500 << 2);
- assertEquals(p2.getPendingBucketCount(), 3 << 2);
- assertEquals(p2.getActiveBucketCount(), 0);
- assertEquals(p2.getBucketCursor(), 503 << 2);
- }
-
- public void testImportInconsistentProgressIncrease() throws ParseException {
- // Bucket progress "file" that upon time of changing from 4 to 7
- // distribution bits and writing the progress had an active bucket
- String input = "VDS bucket progress file\n" +
- "7\n" +
- "32\n" +
- "24\n" +
- "128\n" +
- "100000000000000c:0\n";
- // Now we're at 8 distribution bits
- int db = 8;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken(input);
- assertEquals(7, p.getDistributionBitCount());
- assertEquals(p.getTotalBucketCount(), 1 << 7);
- assertEquals(p.getFinishedBucketCount(), 24);
- // Not yet corrected
- assertEquals(p.getPendingBucketCount(), 1);
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(32, p.getBucketCursor());
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, 1, p);
-
- // Now the range handler should have corrected the progress
- // (but not messed with the distribution bit count)
- assertEquals(7, p.getDistributionBitCount());
- assertEquals(p.getPendingBucketCount(), 1 << 3);
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(24 + (1 << 3), p.getBucketCursor());
-
- iter.setDistributionBitCount(8);
-
- assertEquals(iter.getDistributionBitCount(), 8);
- assertEquals(p.getDistributionBitCount(), 8);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), 8);
-
- assertEquals(p.getTotalBucketCount(), 1 << 8);
- assertEquals(p.getFinishedBucketCount(), 24 << 1);
- assertEquals(p.getPendingBucketCount(), 1 << 4); // Split 4 -> 7 bits, then 7 -> 8 bits
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(p.getBucketCursor(), 24*2 + (1 << 4));
- assertTrue(iter.hasNext());
- }
-
- public void testImportInconsistentProgressDecrease() throws ParseException {
- // Bucket progress "file" that upon time of changing from 4 to 7
- // distribution bits and writing the progress had an active bucket
- String input = "VDS bucket progress file\n" +
- "7\n" +
- "32\n" +
- "24\n" +
- "128\n" +
- "100000000000000c:0\n";
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken(input);
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, 1, p);
-
- assertEquals(iter.getDistributionBitCount(), 7);
- // Now we're at 6 distribution bits
- iter.setDistributionBitCount(6);
-
- assertEquals(iter.getDistributionBitCount(), 6);
- assertEquals(p.getDistributionBitCount(), 6);
- assertEquals(iter.getBucketSource().getDistributionBitCount(), 6);
-
- assertEquals(p.getTotalBucketCount(), 1 << 6);
- assertEquals(p.getFinishedBucketCount(), 24 >> 1);
- assertEquals(p.getPendingBucketCount(), 1 << 2); // Split 4 -> 7 bits, merge 7 -> 6 bits
- assertEquals(p.getActiveBucketCount(), 0);
- assertEquals(p.getBucketCursor(), 24/2 + (1 << 2));
- assertTrue(iter.hasNext());
- }
-
- public void testEntireBucketSpaceCovered() throws ParseException {
- int db = 4;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3];
-
- for (int i = 0; i < 3; ++i) {
- bpp[i] = iter.getNext();
- }
- for (int i = 0; i < 3; ++i) {
- // Must use non-zero progress or all pending will be optimized
- // away by the reset-logic
- iter.update(bpp[i].getSuperbucket(),
- new BucketId(db + 1, bpp[i].getSuperbucket().getId()));
- }
-
- Set<BucketId> buckets = new TreeSet<BucketId>();
- db = 7;
- for (int i = 0; i < (1 << db); ++i) {
- buckets.add(new BucketId(db, i));
- }
-
- iter.setDistributionBitCount(db);
- assertEquals(p.getFinishedBucketCount(), 0);
- assertEquals(p.getPendingBucketCount(), 3 << 3);
-
- // Ensure all buckets are visited once and only once
- while (iter.hasNext()) {
- VisitorIterator.BucketProgress bp = iter.getNext();
- assertTrue(buckets.contains(bp.getSuperbucket()));
- buckets.remove(bp.getSuperbucket());
- }
-
- assertTrue(buckets.isEmpty());
- }
-
- public void testExceptionOnWrongDocumentSelection() throws ParseException {
- BucketIdFactory idFactory = new BucketIdFactory();
- // Since we don't store the actual original document selection in the
- // progress files, we can't really tell whether or not a "wrong" document
- // selection has been given, so we just do a best effort by checking
- // that the number of total buckets match up and that the bucket cursor
- // isn't set for explicit sources
-
- // Try to pass a known document selection to an unknown docsel iterator
- boolean caughtIt = false;
- try {
- ProgressToken p = new ProgressToken("VDS bucket progress file\n16\n3\n1\n3\n"
- + "8000000000001f49:0\n8000000000001a85:0\n");
-
- VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 16, p);
- }
- catch (IllegalArgumentException e) {
- caughtIt = true;
- }
- assertTrue(caughtIt);
-
- // Now try it the other way around
- caughtIt = false;
- try {
- ProgressToken p = new ProgressToken("VDS bucket progress file\n" +
- "10\n" +
- "503\n" +
- "500\n" +
- "1024\n" +
- "28000000000000be:0\n" +
- "28000000000002be:0\n" +
- "28000000000001be:0\n");
-
- VisitorIterator.createFromDocumentSelection("id.group=\"yahoo.com\" or id.user=555", idFactory, 16, p);
- }
- catch (IllegalArgumentException e) {
- caughtIt = true;
- }
- assertTrue(caughtIt);
- }
-
- public void testIsBucketFinished() throws ParseException {
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, 4, p);
-
- assertFalse(p.isBucketFinished(new BucketId(32, 0)));
- // Finish superbucket 0x0000
- iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- assertTrue(p.isBucketFinished(new BucketId(32, 0)));
- // Cursor is 1, but bucket 0x1000 not yet returned
- assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
- VisitorIterator.BucketProgress bp = iter.getNext();
- // Cursor 2, 0x1000 returned but is contained in state, so not finished
- assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3)));
- iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- assertTrue(p.isBucketFinished(new BucketId(32, 1 << 3)));
- // Only superbucket part is used
- assertTrue(p.isBucketFinished(new BucketId(32, 0x12345670))); // ...0000
- assertTrue(p.isBucketFinished(new BucketId(32, 0x12345678))); // ...1000
- assertFalse(p.isBucketFinished(new BucketId(32, 0x12345671))); // ...0001
- assertFalse(p.isBucketFinished(new BucketId(32, 0x12345679))); // ...1001
- }
-
- // Test that altering distribution bit count sets ProgressToken as
- // inconsistent when there are active buckets
- public void testInconsistentState() throws ParseException {
- int db = 16;
- BucketIdFactory idFactory = new BucketIdFactory();
- ProgressToken p = new ProgressToken();
-
- VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
- "id.group != \"yahoo.com\"", idFactory, db, p);
-
- // For this test, have 3 finished bucket, 2 pending and 1 active
- for (int i = 0; i < 3; ++i) {
- iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET);
- }
-
- VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2];
- bpp[0] = iter.getNext();
- bpp[1] = iter.getNext();
- VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active
- iter.update(bpp[0].getSuperbucket(), new BucketId());
- iter.update(bpp[1].getSuperbucket(), new BucketId());
-
- assertFalse(p.isInconsistentState());
- iter.setDistributionBitCount(20);
- assertTrue(p.isInconsistentState());
-
- // Finish active, triggering the consistency fixes
- iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
-
- assertFalse(p.isInconsistentState());
- }
-
- public void testMalformedProgressFile() {
- boolean caughtIt = false;
- try {
- new ProgressToken("VDS bucket progress file\n" +
- "10\n" +
- "503\n" +
- "500\n" +
- "1024\n" +
- "28000000000000be:0\n" +
- "28000000000002be:");
- } catch (IllegalArgumentException e) {
- caughtIt = true;
- }
- assertTrue(caughtIt);
- }
-
- public void testFailOnTooFewLinesInFile() {
- boolean caughtIt = false;
- try {
- new ProgressToken("VDS bucket progress file\n" +
- "10\n" +
- "503\n");
- } catch (IllegalArgumentException e) {
- caughtIt = true;
- }
- assertTrue(caughtIt);
- }
-
- public void testUnknownFirstHeaderLine() {
- boolean caughtIt = false;
- try {
- new ProgressToken("Smurf Time 3000\n" +
- "10\n" +
- "503\n" +
- "500\n" +
- "1024\n" +
- "28000000000000be:0\n" +
- "28000000000002be:0");
- } catch (IllegalArgumentException e) {
- caughtIt = true;
- }
- assertTrue(caughtIt);
- }
-
- public void testBinaryProgressSerialization() {
- String input = "VDS bucket progress file (48.828125% completed)\n" +
- "10\n" +
- "503\n" +
- "500\n" +
- "1024\n" +
- "28000000000000be:0\n" +
- "28000000000002be:0\n" +
- "28000000000001be:0\n";
- ProgressToken p = new ProgressToken(input);
- byte[] buf = p.serialize();
- ProgressToken p2 = new ProgressToken(buf);
- assertEquals(input, p2.toString());
- }
- }
+package com.yahoo.documentapi; + +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorIterator; +import junit.framework.TestCase; +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; + +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.Vector; + +/** + * Tests for VisitorIterator and ProgressToken (kept in one test case because their + * interactions are so tightly coupled) + * @author <a href="mailto:vekterli@yahoo-inc.com">Tor Brede Vekterli</a> + */ +public class VisitorIteratorTestCase extends TestCase { + + public void testIterationSingleBucketUpdate() throws ParseException { + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken progress = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.user = 1234", idFactory, 1, progress); + + assertFalse(progress.hasActive()); + assertEquals(progress.getPendingBucketCount(), 1); + assertEquals(progress.getFinishedBucketCount(), 0); + assertEquals(progress.getTotalBucketCount(), 1); + assertFalse(iter.isDone()); + assertTrue(iter.hasNext()); + assertEquals(iter.getRemainingBucketCount(), 1); + VisitorIterator.BucketProgress b1 = iter.getNext(); + // Upon first getNext of a superbucket, progress == 0 + assertEquals(b1.getSuperbucket(), new BucketId(32, 1234)); + assertEquals(b1.getProgress(), new BucketId()); + assertFalse(iter.hasNext()); + assertFalse(iter.isDone()); + assertEquals(iter.getRemainingBucketCount(), 1); + // Should only be one active bucket; the one we just got + assertEquals(progress.getActiveBucketCount(), 1); + // No pending yet + assertFalse(progress.hasPending()); + // Update the bucket with a sub-bucket, moving it from active to pending + BucketId sub = new BucketId(b1.getSuperbucket().getUsedBits() + 1, b1.getSuperbucket().getId()); + iter.update(b1.getSuperbucket(), sub); + assertFalse(progress.hasActive()); + assertEquals(progress.getPendingBucketCount(), 1); + assertTrue(iter.hasNext()); + assertFalse(iter.isDone()); + assertEquals(iter.getRemainingBucketCount(), 1); + // Get the pending bucket + VisitorIterator.BucketProgress b2 = iter.getNext(); + assertEquals(b2.getSuperbucket(), new BucketId(32, 1234)); + assertEquals(b2.getProgress(), new BucketId(33, 1234)); + assertFalse(iter.hasNext()); + assertEquals(progress.getActiveBucketCount(), 1); + assertFalse(progress.hasPending()); + // Now update with progress==super, signalling that the bucket is done + iter.update(b1.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + assertFalse(progress.hasActive()); + assertFalse(progress.hasPending()); + assertFalse(iter.hasNext()); + assertTrue(iter.isDone()); + assertTrue(progress.isFinished()); + assertEquals(progress.getFinishedBucketCount(), 1); + assertEquals(iter.getRemainingBucketCount(), 0); + } + + public void testProgressSerializationRange() throws ParseException { + int distBits = 4; + + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken progress = new ProgressToken(); + + // docsel will be unknown --> entire bucket range will be covered + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, distBits, progress); + + assertEquals(progress.getDistributionBitCount(), distBits); + assertTrue(iter.getBucketSource() instanceof VisitorIterator.DistributionRangeBucketSource); + + assertEquals(progress.getFinishedBucketCount(), 0); + assertEquals(progress.getTotalBucketCount(), 1 << distBits); + + // First, get+update half of the buckets, marking them as done + long bucketCount = 0; + long bucketStop = 1 << (distBits - 1); + + while (iter.hasNext() && bucketCount != bucketStop) { + VisitorIterator.BucketProgress ids = iter.getNext(); + iter.update(ids.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + ++bucketCount; + } + assertEquals(bucketCount, bucketStop); + // Should be no buckets in limbo at this point + assertFalse(progress.hasActive()); + assertFalse(progress.hasPending()); + assertFalse(iter.isDone()); + assertTrue(iter.hasNext()); + assertEquals(progress.getFinishedBucketCount(), bucketCount); + assertFalse(progress.isFinished()); + + StringBuilder desired = new StringBuilder(); + desired.append("VDS bucket progress file (50.0% completed)\n"); + desired.append(distBits); + desired.append('\n'); + desired.append(bucketCount); // Finished == cursor for this + desired.append('\n'); + desired.append(bucketCount); + desired.append('\n'); + desired.append(1 << distBits); + desired.append('\n'); + + assertEquals(desired.toString(), progress.toString()); + + // Test import, in which case distribution bits are 1 + BucketIdFactory idFactory2 = new BucketIdFactory(); + + // De-serialization with no pending buckets + { + ProgressToken progDs = new ProgressToken(progress.toString()); + + assertEquals(progDs.getDistributionBitCount(), distBits); + assertEquals(progDs.getTotalBucketCount(), 1 << distBits); + assertEquals(progDs.getFinishedBucketCount(), bucketCount); + + VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory2, 1, progDs); + + assertFalse(progDs.hasPending()); + assertFalse(progDs.hasActive()); + assertTrue(iterDs.hasNext()); + assertFalse(iterDs.isDone()); + assertEquals(distBits, iterDs.getDistributionBitCount()); + assertEquals(distBits, progDs.getDistributionBitCount()); + + // Iterator must start up on next bucket in range + VisitorIterator.BucketProgress idDs = iterDs.getNext(); + long resumeKey = ProgressToken.makeNthBucketKey(bucketCount, distBits); + assertEquals(idDs.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(resumeKey))); + assertEquals(idDs.getProgress(), new BucketId()); + } + + // Now fetch a subset of the remaining buckets without finishing them, + // keeping some in the active set and some in pending + int pendingTotal = 1 << (distBits - 3); + int activeTotal = 1 << (distBits - 3); + Vector<VisitorIterator.BucketProgress> buckets = new Vector<VisitorIterator.BucketProgress>(); + + // Pre-fetch, since otherwise we'd reuse pending buckets + for (int i = 0; i < pendingTotal + activeTotal; ++i) { + buckets.add(iter.getNext()); + } + + for (int i = 0; i < pendingTotal + activeTotal; ++i) { + VisitorIterator.BucketProgress idTemp = buckets.get(i); + if (i < activeTotal) { + // Make them 50% done + iter.update(idTemp.getSuperbucket(), + new BucketId(distBits + 2, idTemp.getSuperbucket().getId() | (2 << distBits))); + } + // else: leave hanging as active + } + + assertEquals(progress.getActiveBucketCount(), activeTotal); + assertEquals(progress.getPendingBucketCount(), pendingTotal); + + // we can't reuse the existing string builder, since the bucket cursor + // has changed + desired = new StringBuilder(); + desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n"); + desired.append(distBits); + desired.append('\n'); + desired.append(bucketCount + pendingTotal + activeTotal); + desired.append('\n'); + desired.append(bucketCount); + desired.append('\n'); + desired.append(1 << distBits); + desired.append('\n'); + + assertEquals(progress.getBuckets().entrySet().size(), pendingTotal + activeTotal); + + for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry + : progress.getBuckets().entrySet()) { + desired.append(Long.toHexString(ProgressToken.keyToBucketId(entry.getKey().getKey()))); + desired.append(':'); + desired.append(Long.toHexString(entry.getValue().getProgress().getRawId())); + desired.append('\n'); + } + + assertEquals(progress.toString(), desired.toString()); + + { + // Deserialization with pending buckets + ProgressToken progDs = new ProgressToken(progress.toString()); + + assertEquals(progDs.getDistributionBitCount(), distBits); + assertEquals(progDs.getTotalBucketCount(), 1 << distBits); + assertEquals(progDs.getFinishedBucketCount(), bucketCount); + + VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory2, 1, progDs); + + // All started but nonfinished buckets get placed in pending upon + // deserialization + assertEquals(progDs.getPendingBucketCount(), pendingTotal + activeTotal); + assertEquals(distBits, progDs.getDistributionBitCount()); + assertEquals(distBits, iterDs.getDistributionBitCount()); + assertFalse(progDs.hasActive()); + assertTrue(iterDs.hasNext()); + assertFalse(iterDs.isDone()); + assertEquals(progDs.getBucketCursor(), bucketCount + pendingTotal + activeTotal); + } + + // Finish all the active buckets + for (int i = activeTotal; i < activeTotal + pendingTotal; ++i) { + iter.update(buckets.get(i).getSuperbucket(), ProgressToken.FINISHED_BUCKET); + ++bucketCount; + } + + assertEquals(progress.getActiveBucketCount(), 0); + boolean consistentNext = true; + // Get all pending/remaining sourced and finish them all + while (!iter.isDone()) { + if (!iter.hasNext()) { + consistentNext = false; + break; + } + VisitorIterator.BucketProgress bp = iter.getNext(); + iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + ++bucketCount; + } + + assertTrue(consistentNext); + assertFalse(iter.hasNext()); + assertTrue(progress.isFinished()); + // Cumulative number of finished buckets must match 2^distbits + assertEquals(bucketCount, 1 << distBits); + StringBuilder finished = new StringBuilder(); + finished.append("VDS bucket progress file (100.0% completed)\n"); + finished.append(distBits); + finished.append('\n'); + finished.append(1 << distBits); // Cursor + finished.append('\n'); + finished.append(1 << distBits); // Finished + finished.append('\n'); + finished.append(1 << distBits); // Total + finished.append('\n'); + + assertEquals(progress.toString(), finished.toString()); + } + + public void testProgressSerializationExplicit() throws ParseException { + int distBits = 16; + + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken progress = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, progress); + + assertEquals(progress.getDistributionBitCount(), distBits); + assertTrue(iter.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource); + + assertEquals(progress.getFinishedBucketCount(), 0); + assertEquals(progress.getTotalBucketCount(), 3); + assertEquals(progress.getPendingBucketCount(), 3); + + VisitorIterator.BucketProgress bp1 = iter.getNext(); + VisitorIterator.BucketProgress bp2 = iter.getNext(); + assertEquals(progress.getPendingBucketCount(), 1); + assertEquals(progress.getActiveBucketCount(), 2); + // Buckets are ordered by their reverse bucket id key + assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234)); + assertEquals(bp1.getProgress(), new BucketId()); + // Put bucket 1234 back into pending + iter.update(bp1.getSuperbucket(), new BucketId(36, 1234)); + assertEquals(progress.getPendingBucketCount(), 2); + + assertEquals(bp2.getSuperbucket(), new BucketId(32, 8009)); + assertEquals(bp2.getProgress(), new BucketId()); + + { + StringBuilder desired = new StringBuilder(); + desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n"); + desired.append(distBits); + desired.append('\n'); + desired.append(0); + desired.append('\n'); + desired.append(0); + desired.append('\n'); + desired.append(3); + desired.append('\n'); + // Pending/active buckets are written in an increasing (key, not + // bucket-id!) order + desired.append(Long.toHexString(new BucketId(32, 1234).getRawId())); + desired.append(':'); + desired.append(Long.toHexString(new BucketId(36, 1234).getRawId())); + desired.append('\n'); + desired.append(Long.toHexString(new BucketId(32, 8009).getRawId())); + desired.append(":0\n"); + desired.append(Long.toHexString(new BucketId(32, 6789).getRawId())); + desired.append(":0\n"); + + assertEquals(desired.toString(), progress.toString()); + + ProgressToken prog2 = new ProgressToken(progress.toString()); + assertEquals(prog2.getDistributionBitCount(), distBits); + assertEquals(prog2.getTotalBucketCount(), 3); + assertEquals(prog2.getFinishedBucketCount(), 0); + + VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection( + "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, prog2); + + assertEquals(prog2.getPendingBucketCount(), 3); + assertFalse(prog2.hasActive()); + assertTrue(iter2.hasNext()); + assertFalse(iter2.isDone()); + + assertTrue(iter2.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource); + assertFalse(iter2.getBucketSource().hasNext()); + + VisitorIterator.BucketProgress bp = iter2.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(32, 1234)); + assertEquals(bp.getProgress(), new BucketId(36, 1234)); + assertEquals(prog2.getPendingBucketCount(), 2); + + assertTrue(iter2.hasNext()); + assertFalse(iter2.isDone()); + bp = iter2.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(32, 8009)); + assertEquals(bp.getProgress(), new BucketId()); + assertEquals(prog2.getPendingBucketCount(), 1); + + assertTrue(iter2.hasNext()); + assertFalse(iter2.isDone()); + bp = iter2.getNext(); + assertEquals(prog2.getPendingBucketCount(), 0); + assertEquals(bp.getSuperbucket(), new BucketId(32, 6789)); + assertEquals(bp.getProgress(), new BucketId()); + assertFalse(iter2.hasNext()); + assertFalse(iter2.isDone()); // Active buckets + assertEquals(prog2.getActiveBucketCount(), 3); + } + + // Finish off all active buckets + assertTrue(iter.hasNext()); + assertFalse(iter.isDone()); + bp1 = iter.getNext(); + assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234)); + assertEquals(bp1.getProgress(), new BucketId(36, 1234)); + + iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + + assertTrue(iter.hasNext()); + assertFalse(iter.isDone()); + bp1 = iter.getNext(); + assertEquals(bp1.getSuperbucket(), new BucketId(32, 6789)); + assertEquals(bp1.getProgress(), new BucketId()); + + // Just to make sure Java serializes the long properly + assertEquals( + progress.toString(), + "VDS bucket progress file (" + progress.percentFinished() + "% completed)\n" + + "16\n" + + "0\n" + + "1\n" + + "3\n" + + "8000000000001f49:0\n" + + "8000000000001a85:0\n"); + + iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + + // At this point, we've got one active but no pending, so hasNext == false, + // but isDone is also == false + assertFalse(iter.hasNext()); + assertFalse(iter.isDone()); + assertEquals(progress.getPendingBucketCount(), 0); + assertEquals(progress.getActiveBucketCount(), 1); + + iter.update(bp2.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + assertFalse(iter.hasNext()); + assertTrue(iter.isDone()); + assertTrue(progress.isFinished()); + assertEquals(progress.getActiveBucketCount(), 0); + + { + StringBuilder finished = new StringBuilder(); + finished.append("VDS bucket progress file (100.0% completed)\n"); + finished.append(distBits); + finished.append('\n'); + finished.append(0); // Cursor (not used by explicit) + finished.append('\n'); + finished.append(3); // Finished + finished.append('\n'); + finished.append(3); // Total + finished.append('\n'); + + assertEquals(finished.toString(), progress.toString()); + } + } + + /** + * Test that doing update() on a bucket several times in a row (without re-fetching + * from getNext first) works + * @throws ParseException + */ + public void testActiveUpdate() throws ParseException { + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken progress = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group = \"yahoo.com\"", idFactory, 16, progress); + + VisitorIterator.BucketProgress bp = iter.getNext(); + + assertEquals(progress.getPendingBucketCount(), 0); + assertEquals(progress.getActiveBucketCount(), 1); + + BucketId superbucket = bp.getSuperbucket(); + int usedBits = superbucket.getUsedBits(); + + iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (2L << usedBits))); + assertEquals(progress.getPendingBucketCount(), 1); + assertEquals(progress.getActiveBucketCount(), 0); + iter.update(superbucket, new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits))); + assertEquals(progress.getPendingBucketCount(), 1); + assertEquals(progress.getActiveBucketCount(), 0); + + bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), superbucket); + assertEquals(bp.getProgress(), new BucketId(usedBits + 2, superbucket.getId() | (1L << usedBits))); + assertEquals(progress.getPendingBucketCount(), 0); + assertEquals(progress.getActiveBucketCount(), 1); + } + + /** + * Test that ensures doing update(superbucket, 0) simply puts the bucket back in + * pending + * @throws ParseException + */ + public void testNullAndSuperUpdate() throws ParseException { + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken progress = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group = \"yahoo.com\"", idFactory, 16, progress); + + assertEquals(progress.getPendingBucketCount(), 1); + + VisitorIterator.BucketProgress bp = iter.getNext(); + assertEquals(bp.getProgress(), new BucketId()); + BucketId superbucket = bp.getSuperbucket(); + BucketId sub = bp.getProgress(); + + assertFalse(iter.hasNext()); + assertFalse(iter.isDone()); + assertEquals(progress.getPendingBucketCount(), 0); + assertEquals(progress.getActiveBucketCount(), 1); + + // 0-bucket + iter.update(superbucket, ProgressToken.NULL_BUCKET); + assertTrue(iter.hasNext()); + assertFalse(iter.isDone()); + assertEquals(progress.getPendingBucketCount(), 1); + assertEquals(progress.getActiveBucketCount(), 0); + + VisitorIterator.BucketProgress bp2 = iter.getNext(); + assertEquals(bp2.getSuperbucket(), superbucket); + assertEquals(bp2.getProgress(), ProgressToken.NULL_BUCKET); + assertEquals(progress.getPendingBucketCount(), 0); + assertEquals(progress.getActiveBucketCount(), 1); + + // progress == super + iter.update(superbucket, superbucket); + assertTrue(iter.hasNext()); + assertFalse(iter.isDone()); + assertEquals(progress.getPendingBucketCount(), 1); + assertEquals(progress.getActiveBucketCount(), 0); + + bp2 = iter.getNext(); + assertEquals(bp2.getSuperbucket(), superbucket); + assertEquals(bp2.getProgress(), superbucket); + assertEquals(progress.getPendingBucketCount(), 0); + assertEquals(progress.getActiveBucketCount(), 1); + } + + public void testDeserializedFinishedProgress() { + StringBuilder finished = new StringBuilder(); + finished.append("VDS bucket progress file\n"); // legacy; no completion percentage + finished.append(17); + finished.append('\n'); + finished.append(1L << 17); // Cursor + finished.append('\n'); + finished.append(1L << 17); // Finished + finished.append('\n'); + finished.append(1L << 17); // Total + finished.append('\n'); + + ProgressToken token = new ProgressToken(finished.toString()); + assertEquals(token.getDistributionBitCount(), 17); + assertEquals(token.getTotalBucketCount(), 1L << 17); + assertEquals(token.getFinishedBucketCount(), 1L << 17); + assertEquals(token.getBucketCursor(), 1L << 17); + assertTrue(token.isFinished()); + + ProgressToken token2 = new ProgressToken(token.serialize()); + assertEquals(17, token2.getDistributionBitCount()); + assertEquals(1L << 17, token2.getTotalBucketCount()); + assertEquals(1L << 17, token2.getFinishedBucketCount()); + assertEquals(1L << 17, token2.getBucketCursor()); + assertTrue(token2.isFinished()); + } + + public void testBucketProgressFraction() { + double epsilon = 0.00001; + // No progress + BucketId b_0 = new BucketId(); + // No split; only superbucket (100%) + BucketId b_100_0 = new BucketId(16, 1234); + // 1 split (1/2) + BucketId b_50_1 = new BucketId(17, 1234); + BucketId b_100_1 = new BucketId(17, 1234 | (1 << 16)); + // 2 splits (1/4) + BucketId b_25_2 = new BucketId(18, 1234); + BucketId b_50_2 = new BucketId(18, 1234 | (2 << 16)); + BucketId b_75_2 = new BucketId(18, 1234 | (1 << 16)); + BucketId b_100_2 = new BucketId(18, 1234 | (3 << 16)); + + ProgressToken p = new ProgressToken(16); + + BucketId sb = new BucketId(16, 1234); + + assertEquals(p.progressFraction(new BucketId(32, 1234), b_0), 0.0, epsilon); + + assertEquals(p.progressFraction(sb, b_100_0), 1.0, epsilon); + + assertEquals(p.progressFraction(sb, b_50_1), 0.5, epsilon); + assertEquals(p.progressFraction(sb, b_100_1), 1.0, epsilon); + + assertEquals(p.progressFraction(sb, b_25_2), 0.25, epsilon); + assertEquals(p.progressFraction(sb, b_50_2), 0.5, epsilon); + assertEquals(p.progressFraction(sb, b_75_2), 0.75, epsilon); + assertEquals(p.progressFraction(sb, b_100_2), 1.0, epsilon); + + assertEquals(p.progressFraction(new BucketId(0x8000000000000000L), + new BucketId(0xb0000fff00000000L)), 1.0, epsilon); + } + + public void testProgressEstimation() throws ParseException { + int distBits = 4; + + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken progress = new ProgressToken(); + + // Create a range of [0, 16) superbuckets + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, distBits, progress); + + assertEquals(progress.getDistributionBitCount(), 4); + + double epsilon = 0.00001; + assertEquals(progress.percentFinished(), 0, epsilon); + VisitorIterator.BucketProgress bp = iter.getNext(); + // Finish first superbucket (6.25% total) + iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + assertEquals(progress.percentFinished(), 6.25, epsilon); + assertEquals(progress.getFinishedBucketCount(), 1); + + bp = iter.getNext(); + VisitorIterator.BucketProgress bp3 = iter.getNext(); + VisitorIterator.BucketProgress bp4 = iter.getNext(); + + // Finish second (12.5% total) + iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + assertEquals(progress.percentFinished(), 12.5, epsilon); + assertEquals(progress.getFinishedBucketCount(), 2); + + // Finish third bucket 75% through (17.1875% total) + iter.update(bp3.getSuperbucket(), new BucketId(distBits + 2, bp3.getSuperbucket().getId() | (1 << distBits))); + assertEquals(progress.percentFinished(), 17.1875, epsilon); + assertEquals(progress.getFinishedBucketCount(), 2); + + // Finish fourth bucket 25% through (18.75% total) + iter.update(bp4.getSuperbucket(), new BucketId(distBits + 2, bp4.getSuperbucket().getId())); + assertEquals(progress.percentFinished(), 18.75, epsilon); + assertEquals(progress.getFinishedBucketCount(), 2); + // Finish all buckets + iter.update(bp4.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + iter.update(bp3.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + assertEquals(progress.percentFinished(), 25, epsilon); + assertEquals(progress.getFinishedBucketCount(), 4); + + while (iter.hasNext()) { + bp = iter.getNext(); + iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + } + + assertEquals(progress.getFinishedBucketCount(), 16); + assertEquals(progress.percentFinished(), 100, epsilon); + } + + public void testBucketKeyWrapperOrdering() { + ProgressToken.BucketKeyWrapper bk1 = new ProgressToken.BucketKeyWrapper(0x0000000000000001L); + ProgressToken.BucketKeyWrapper bk2 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL); + ProgressToken.BucketKeyWrapper bk3 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L); + ProgressToken.BucketKeyWrapper bk4 = new ProgressToken.BucketKeyWrapper(0xFFFFFFFFFFFFFFFFL); + assertTrue(bk1.compareTo(bk2) < 0); + assertTrue(bk2.compareTo(bk3) < 0); + assertTrue(bk3.compareTo(bk4) < 0); + assertTrue(bk2.compareTo(bk1) > 0); + assertTrue(bk3.compareTo(bk2) > 0); + assertTrue(bk4.compareTo(bk3) > 0); + ProgressToken.BucketKeyWrapper bk5 = new ProgressToken.BucketKeyWrapper(0x7FFFFFFFFFFFFFFFL); + ProgressToken.BucketKeyWrapper bk6 = new ProgressToken.BucketKeyWrapper(0x8000000000000000L); + assertTrue(bk5.compareTo(bk2) == 0); + assertTrue(bk6.compareTo(bk3) == 0); + } + + private void doTestBucketKeyGeneration(int db) { + // Can't use longs since they won't sort properly when MSB is set + ProgressToken.BucketKeyWrapper[] keys = new ProgressToken.BucketKeyWrapper[1 << db]; + + // Generate entire bucket space for db + for (int i = 0; i < (1 << db); ++i) { + keys[i] = new ProgressToken.BucketKeyWrapper( + ProgressToken.bucketToKey(new BucketId(db, i).getId())); + } + Arrays.sort(keys); + + boolean consistentKeys = true; + // Verify that makeNthBucketKey yields the same result as the equivalent + // ordered value in the array of keys + for (int i = 0; i < (1 << db); ++i) { + long genKey = ProgressToken.makeNthBucketKey(i, db); + long knownKey = keys[i].getKey(); + if (genKey != knownKey) { + consistentKeys = false; + break; + } + } + assertTrue(consistentKeys); + } + + public void testBucketKeyGeneration() { + // Due to the number of objects needed to be allocated, only test for a + // small set of distribution bits + for (int i = 1; i < 14; ++i) { + doTestBucketKeyGeneration(i); + } + } + + public void testSingleBucketSplits() throws ParseException { + int db = 2; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + // Create a range of [0, 4) superbuckets + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + VisitorIterator.BucketProgress bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db, 0)); + // Put back as pending + iter.update(bp.getSuperbucket(), new BucketId()); + assertEquals(p.getPendingBucketCount(), 1); + p.splitPendingBucket(new BucketId(db, 0)); + assertEquals(p.getPendingBucketCount(), 2); + bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 0)); // left split + assertEquals(bp.getProgress(), new BucketId(0)); + bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 4)); // right split + assertEquals(bp.getProgress(), new BucketId(0)); + + bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db, 2)); + // Put back as pending, with a progress of 10010. This implies splitting + // the bucket should set both splits with a progress to 10010 + iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x12)); + assertEquals(p.getPendingBucketCount(), 1); + p.splitPendingBucket(new BucketId(db, 2)); + assertEquals(p.getPendingBucketCount(), 2); + bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 2)); // left split + assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12)); + bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 6)); // right split + assertEquals(bp.getProgress(), new BucketId(db + 3, 0x12)); + + bp = iter.getNext(); + // Put back as pending with a progress of 10101. This implies splitting the + // bucket should _discard_ left and set right's progress to 10101. + // Update: no it shouldn't, we now split with equal progress without + // discarding + assertEquals(bp.getSuperbucket(), new BucketId(db, 1)); + iter.update(bp.getSuperbucket(), new BucketId(db + 3, 0x15)); + assertEquals(p.getPendingBucketCount(), 1); + p.splitPendingBucket(new BucketId(db, 1)); + assertEquals(p.getPendingBucketCount(), 2); + bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 1)); + assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15)); + bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db + 1, 5)); // right split + assertEquals(bp.getProgress(), new BucketId(db + 3, 0x15)); + } + + /** + * Test increasing the distribution bits for a full bucket space range + * source with no finished, active or pending buckets + * @throws ParseException upon docsel parse failure (shouldn't happen) + */ + public void testRangeDistributionBitIncrease1NoPending() throws ParseException { + int db = 2; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + // Test for empty progress token. no splitting involved + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + assertEquals(p.getTotalBucketCount(), 4); + iter.setDistributionBitCount(db + 1); + assertEquals(p.getTotalBucketCount(), 8); + assertEquals(p.getDistributionBitCount(), db + 1); + assertEquals(iter.getDistributionBitCount(), db + 1); + assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1); + + int[] desired = new int[] { 0, 4, 2, 6, 1, 5, 3, 7 }; + for (int i = 0; i < 8; ++i) { + VisitorIterator.BucketProgress bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db + 1, desired[i])); + } + } + + public void testRangeDistributionBitIncrease1AllBucketStates() throws ParseException { + int db = 3; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + // For this test, have 1 finished bucket, 3 pending and 0 active (we + // want to have the splitting to be triggered immediately) + VisitorIterator.BucketProgress bp = iter.getNext(); + iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3]; + bpp[0] = iter.getNext(); + bpp[1] = iter.getNext(); + bpp[2] = iter.getNext(); + iter.update(bpp[0].getSuperbucket(), new BucketId()); + iter.update(bpp[1].getSuperbucket(), new BucketId()); + iter.update(bpp[2].getSuperbucket(), new BucketId()); + + assertEquals(p.getFinishedBucketCount(), 1); + assertEquals(p.getPendingBucketCount(), 3); + assertEquals(p.getActiveBucketCount(), 0); + + iter.setDistributionBitCount(db + 1); + + assertEquals(p.getTotalBucketCount(), 16); + assertEquals(p.getFinishedBucketCount(), 2); + assertEquals(p.getPendingBucketCount(), 6); + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(p.getDistributionBitCount(), db + 1); + assertEquals(iter.getDistributionBitCount(), db + 1); + assertEquals(iter.getBucketSource().getDistributionBitCount(), db + 1); + + // Bucket 3:0x4 -> 4:0x4 & 4:0xC + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x04)); + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0C)); + // Bucket 3:0x2 -> 4:0x2 & 4:0xA + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x02)); + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0A)); + // Bucket 3:0x6 -> 4:0x6 & 4:0xE + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x06)); + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0E)); + + assertEquals(p.getPendingBucketCount(), 0); + // Bucket source should now begin returning from bucket 4:0x1 + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x01)); + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x09)); + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x05)); + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x0D)); + assertEquals(iter.getNext().getSuperbucket(), new BucketId(db + 1, 0x03)); + // Assume correct from here on + } + + public void testRangeDistributionIncreaseMultipleBits() throws ParseException { + int db = 16; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + // For this test, have 3 finished bucket, 2 pending and 1 active + for (int i = 0; i < 3; ++i) { + iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET); + } + + VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2]; + bpp[0] = iter.getNext(); + bpp[1] = iter.getNext(); + VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active + iter.update(bpp[0].getSuperbucket(), new BucketId()); + iter.update(bpp[1].getSuperbucket(), new BucketId()); + + iter.setDistributionBitCount(20); + // ProgressToken doesn't change yet, since it had active buckets + assertEquals(p.getDistributionBitCount(), 16); + assertEquals(iter.getDistributionBitCount(), 20); + assertEquals(iter.getBucketSource().getDistributionBitCount(), 20); + + assertFalse(iter.hasNext()); + assertFalse(iter.isDone()); + assertTrue(iter.getBucketSource().shouldYield()); + assertEquals(p.getPendingBucketCount(), 2); + assertEquals(p.getActiveBucketCount(), 1); + + // Finish active, triggering the consistency fixes + iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + + assertEquals(p.getDistributionBitCount(), 20); + assertEquals(p.getPendingBucketCount(), 32); + assertEquals(p.getActiveBucketCount(), 0); + // Each bucket with db:16 becomes equal to 16 buckets with db:20, so + // the bucket space position must be 16 * 6 = 96 + assertEquals(p.getBucketCursor(), 96); + // Each finished bucket also covers less ground, so count is upped + // accordingly + assertEquals(p.getFinishedBucketCount(), 16 * 4); + + // Remove pending that came from the split + // Bucket space that should be covered by the 32 buckets is [48, 80) + // when using 20 distribution bits + for (int i = 0; i < 32; ++i) { + long testKey = ProgressToken.makeNthBucketKey(i + 48, 20); + VisitorIterator.BucketProgress bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(testKey))); + iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + } + assertEquals(p.getPendingBucketCount(), 0); + assertEquals(p.getFinishedBucketCount(), 16 * 6); + + // Bucket source should now begin returning from bucket 20:0x6000 + assertEquals(iter.getNext().getSuperbucket(), new BucketId(20, 0x6000)); + } + + public void testSingleBucketMerge() throws ParseException { + int db = 2; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + // Create a range of [0, 4) superbuckets + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + VisitorIterator.BucketProgress bp = iter.getNext(); + // Put back as pending and split it + iter.update(bp.getSuperbucket(), new BucketId()); + p.splitPendingBucket(new BucketId(db, 0)); + assertEquals(p.getPendingBucketCount(), 2); + // Merge both back into one node. Merge from left sibling with right present + p.mergePendingBucket(new BucketId(db + 1, 0)); + assertEquals(p.getPendingBucketCount(), 1); + bp = iter.getNext(); + assertEquals(bp.getSuperbucket(), new BucketId(db, 0)); + } + + public void testRangeDistributionBitDecrease1() throws ParseException { + int db = 16; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + VisitorIterator.DistributionRangeBucketSource src + = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource(); + + assertTrue(src.isLosslessResetPossible()); + + // For this test, have 3 finished buckets, 6 pending and 1 active + // This gives a sibling "distribution" of FF FP PP PP PA. When all + // active buckets have been updated, 3 merges should be triggered + for (int i = 0; i < 3; ++i) { + iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET); + } + + assertFalse(src.isLosslessResetPossible()); + + VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[6]; + for (int i = 0; i < 6; ++i) { + bpp[i] = iter.getNext(); + } + VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active + for (int i = 0; i < 6; ++i) { + iter.update(bpp[i].getSuperbucket(), new BucketId()); + } + + assertEquals(p.getBucketCursor(), 10); + + iter.setDistributionBitCount(db - 1); + assertEquals(iter.getDistributionBitCount(), db - 1); + assertEquals(p.getDistributionBitCount(), db); + assertEquals(iter.getBucketSource().getDistributionBitCount(), db - 1); + // The iterator is waiting patiently for all active buckets to be updated, + // at which point it will performed the merging and actually updating the + // progress token's distribution bit count + assertTrue(iter.getBucketSource().shouldYield()); + assertFalse(iter.hasNext()); + assertFalse(iter.isDone()); + assertEquals(p.getActiveBucketCount(), 1); + iter.update(bpa.getSuperbucket(), new BucketId()); + + assertEquals(p.getDistributionBitCount(), db - 1); + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(p.getPendingBucketCount(), 4); // 3 merges, P PP PP PP -> P P P P + + assertEquals(p.getFinishedBucketCount(), 1); + assertEquals(p.getBucketCursor(), 5); + } + + // Test that splitting and merging from and to the same db count gives + // back the initial state + public void testRangeDistributionBitIncreaseDecrease() throws ParseException { + int db = 16; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + VisitorIterator.DistributionRangeBucketSource src + = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource(); + + assertTrue(src.isLosslessResetPossible()); + + // "Sabotage" resetting by having at least 1 finished + iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET); + + VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4]; + for (int i = 0; i < 4; ++i) { + bpp[i] = iter.getNext(); + } + for (int i = 0; i < 4; ++i) { + iter.update(bpp[i].getSuperbucket(), new BucketId()); + } + + assertFalse(src.isLosslessResetPossible()); + + iter.setDistributionBitCount(20); + assertEquals(p.getDistributionBitCount(), 20); + assertEquals(p.getPendingBucketCount(), 4 << 4); + assertFalse(iter.getBucketSource().shouldYield()); + assertEquals(p.getBucketCursor(), 5 << 4); + + iter.setDistributionBitCount(16); + + assertEquals(p.getDistributionBitCount(), 16); + assertEquals(p.getPendingBucketCount(), 4); + assertFalse(iter.getBucketSource().shouldYield()); + assertEquals(p.getBucketCursor(), 5); + } + + // Test that intermittent changes in distribution are handled properly, e.g. + // changing from 11 -> 9 with X active and then before all those are flushed, + // the distribution goes up to 12 + public void testRangeDistributionBitChangeWithoutDone() throws ParseException { + int db = 11; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + VisitorIterator.DistributionRangeBucketSource src + = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource(); + + VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[4]; + for (int i = 0; i < 4; ++i) { + bpp[i] = iter.getNext(); + } + for (int i = 0; i < 2; ++i) { + iter.update(bpp[i].getSuperbucket(), new BucketId()); + } + + assertFalse(src.isLosslessResetPossible()); + + // Now 2 pending, 2 active + + iter.setDistributionBitCount(9); + assertEquals(p.getDistributionBitCount(), 11); + assertEquals(p.getActiveBucketCount(), 2); + assertEquals(p.getPendingBucketCount(), 2); + assertTrue(iter.getBucketSource().shouldYield()); + // Update as pending, still with old count since there's 1 more active + // with bpp[2]. Have progress so that lossless reset isn't possible + iter.update(bpp[3].getSuperbucket(), new BucketId(15, bpp[3].getSuperbucket().getId())); + + iter.setDistributionBitCount(12); + assertEquals(p.getActiveBucketCount(), 1); + assertEquals(p.getPendingBucketCount(), 3); + assertTrue(iter.getBucketSource().shouldYield()); + + // Serialize before token is updated to 12 bits + String serialized = p.toString(); + + iter.update(bpp[2].getSuperbucket(), ProgressToken.FINISHED_BUCKET); + + assertEquals(p.getActiveBucketCount(), 0); + // All active buckets are at db=11, so they should be split once each + assertEquals(p.getPendingBucketCount(), 3 * 2); + assertFalse(iter.getBucketSource().shouldYield()); + assertEquals(p.getFinishedBucketCount(), 2); + + // Ensure we get a consistent progress token imported + ProgressToken p2 = new ProgressToken(serialized); + assertEquals(p2.getDistributionBitCount(), 11); // Not yet updated + + BucketIdFactory idFactory2 = new BucketIdFactory(); + VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory2, 1, p2); + + // Not yet updated, since we don't trust the initial BucketIdFactory + assertEquals(iter2.getDistributionBitCount(), 11); + assertEquals(p2.getDistributionBitCount(), 11); + iter2.setDistributionBitCount(12); + // Now it has been updated + assertEquals(p2.getDistributionBitCount(), 12); + assertEquals(p2.getPendingBucketCount(), 8); + assertEquals(p2.getBucketCursor(), 8); + assertEquals(p2.getFinishedBucketCount(), 0); + } + + // Test a drop from 31->11 bits upon iteration start + public void testRangeDistributionBitInitialDrop() throws ParseException { + int db = 31; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[3]; + bp[0] = iter.getNext(); + bp[1] = iter.getNext(); + bp[2] = iter.getNext(); + iter.update(bp[2].getSuperbucket(), new BucketId()); + iter.update(bp[1].getSuperbucket(), new BucketId()); + assertEquals(p.getActiveBucketCount(), 1); + + iter.setDistributionBitCount(11); + + assertFalse(iter.hasNext()); + assertFalse(iter.isDone()); + assertEquals(p.getActiveBucketCount(), 1); + + // Updating the active bucket allows the merging to take place + iter.update(new BucketId(31, 0), new BucketId()); + + assertTrue(iter.hasNext()); + assertFalse(iter.isDone()); + + // All pending buckets should have been merged down to just 1 now + // Update: now rather gets reset + assertEquals(p.getPendingBucketCount(), 0); + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(p.getFinishedBucketCount(), 0); + assertEquals(p.getBucketCursor(), 0); + + bp[0] = iter.getNext(); + assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0)); + } + + // Similar to testRangeDistributionBitInitialDrop, but going from 1 to 11 + // This tests that doing so may be done in an optimized way rather than + // attempting to split enough buckets to cover the entire bucket space! + public void testRangeDistributionLosslessReset() throws ParseException { + int db = 1; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + VisitorIterator.DistributionRangeBucketSource src + = (VisitorIterator.DistributionRangeBucketSource)iter.getBucketSource(); + + VisitorIterator.BucketProgress[] bp = new VisitorIterator.BucketProgress[2]; + bp[0] = iter.getNext(); + bp[1] = iter.getNext(); + + String serialized = p.toString(); + + assertFalse(src.isLosslessResetPossible()); + + iter.update(bp[1].getSuperbucket(), new BucketId()); + assertEquals(p.getActiveBucketCount(), 1); + + iter.setDistributionBitCount(11); + + assertFalse(src.isLosslessResetPossible()); + assertEquals(p.getDistributionBitCount(), 1); // Still at 1 + + assertFalse(iter.hasNext()); + assertFalse(iter.isDone()); + assertEquals(p.getActiveBucketCount(), 1); + + // Updating the active bucket allows the reset to take place + iter.update(new BucketId(1, 0), new BucketId()); + + assertTrue(iter.hasNext()); + assertFalse(iter.isDone()); + + // Should not be any buckets pending/active and the cursor should be + // back at 0 + assertEquals(p.getPendingBucketCount(), 0); + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(p.getFinishedBucketCount(), 0); + assertEquals(p.getBucketCursor(), 0); + assertEquals(p.getDistributionBitCount(), 11); + + bp[0] = iter.getNext(); + assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0)); + + // Ensure resetting also works when you're importing existing + // progress + p = new ProgressToken(serialized); + idFactory = new BucketIdFactory(); + iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, 1, p); + + iter.setDistributionBitCount(11); + + assertEquals(p.getPendingBucketCount(), 0); + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(p.getFinishedBucketCount(), 0); + assertEquals(p.getBucketCursor(), 0); + assertEquals(p.getDistributionBitCount(), 11); + + bp[0] = iter.getNext(); + assertEquals(bp[0].getSuperbucket(), new BucketId(11, 0)); + } + + public void testExplicitDistributionBitIncrease() throws ParseException { + int distBits = 12; + + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p); + + assertEquals(iter.getDistributionBitCount(), distBits); + assertEquals(p.getDistributionBitCount(), distBits); + assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits); + + iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET); + iter.setDistributionBitCount(16); + + assertEquals(iter.getDistributionBitCount(), 16); + assertEquals(p.getDistributionBitCount(), 16); + assertEquals(iter.getBucketSource().getDistributionBitCount(), 16); + // Changing dist bits for explicit source should change nothing + assertEquals(p.getPendingBucketCount(), 2); + assertEquals(p.getFinishedBucketCount(), 1); + assertEquals(p.getTotalBucketCount(), 3); + } + + public void testExplicitDistributionBitDecrease() throws ParseException { + int distBits = 20; + + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, p); + + assertEquals(iter.getDistributionBitCount(), distBits); + assertEquals(p.getDistributionBitCount(), distBits); + assertEquals(iter.getBucketSource().getDistributionBitCount(), distBits); + + iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET); + iter.setDistributionBitCount(16); + + assertEquals(iter.getDistributionBitCount(), 16); + assertEquals(p.getDistributionBitCount(), 16); + assertEquals(iter.getBucketSource().getDistributionBitCount(), 16); + // Changing dist bits for explicit source should change nothing + assertEquals(p.getPendingBucketCount(), 2); + assertEquals(p.getFinishedBucketCount(), 1); + assertEquals(p.getTotalBucketCount(), 3); + } + + public void testExplicitDistributionImportNoTruncation() throws ParseException { + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, 20, p); + assertEquals(20, iter.getDistributionBitCount()); + assertEquals(20, p.getDistributionBitCount()); + assertEquals(20, iter.getBucketSource().getDistributionBitCount()); + + iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET); + + // Make sure no truncation is done on import + String serialized = p.toString(); + ProgressToken p2 = new ProgressToken(serialized); + BucketIdFactory idFactory2 = new BucketIdFactory(); + VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection( + "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory2, 1, p2); + assertEquals(20, iter2.getDistributionBitCount()); + assertEquals(20, p2.getDistributionBitCount()); + assertEquals(20, iter2.getBucketSource().getDistributionBitCount()); + assertEquals(2, p2.getPendingBucketCount()); + assertEquals(1, p2.getFinishedBucketCount()); + assertEquals(3, p2.getTotalBucketCount()); + } + + public void testImportProgressWithOutdatedDistribution() throws ParseException { + String input = "VDS bucket progress file\n" + + "10\n" + + "503\n" + + "500\n" + + "1024\n" + + "28000000000000be:0\n" + + "28000000000002be:0\n" + + "28000000000001be:0\n"; + + int db = 12; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(input); + assertEquals(10, p.getDistributionBitCount()); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, 1, p); + + iter.setDistributionBitCount(12); + assertEquals(iter.getDistributionBitCount(), 12); + assertEquals(p.getDistributionBitCount(), 12); + assertEquals(iter.getBucketSource().getDistributionBitCount(), 12); + + assertEquals(p.getTotalBucketCount(), 1 << 12); + assertEquals(p.getFinishedBucketCount(), 500 << 2); + assertEquals(p.getPendingBucketCount(), 3 << 2); + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(p.getBucketCursor(), 503 << 2); + assertTrue(iter.hasNext()); + + ProgressToken p2 = new ProgressToken(p.serialize()); + assertEquals(p2.getDistributionBitCount(), 12); + assertEquals(p2.getTotalBucketCount(), 1 << 12); + assertEquals(p2.getFinishedBucketCount(), 500 << 2); + assertEquals(p2.getPendingBucketCount(), 3 << 2); + assertEquals(p2.getActiveBucketCount(), 0); + assertEquals(p2.getBucketCursor(), 503 << 2); + } + + public void testImportInconsistentProgressIncrease() throws ParseException { + // Bucket progress "file" that upon time of changing from 4 to 7 + // distribution bits and writing the progress had an active bucket + String input = "VDS bucket progress file\n" + + "7\n" + + "32\n" + + "24\n" + + "128\n" + + "100000000000000c:0\n"; + // Now we're at 8 distribution bits + int db = 8; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(input); + assertEquals(7, p.getDistributionBitCount()); + assertEquals(p.getTotalBucketCount(), 1 << 7); + assertEquals(p.getFinishedBucketCount(), 24); + // Not yet corrected + assertEquals(p.getPendingBucketCount(), 1); + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(32, p.getBucketCursor()); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, 1, p); + + // Now the range handler should have corrected the progress + // (but not messed with the distribution bit count) + assertEquals(7, p.getDistributionBitCount()); + assertEquals(p.getPendingBucketCount(), 1 << 3); + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(24 + (1 << 3), p.getBucketCursor()); + + iter.setDistributionBitCount(8); + + assertEquals(iter.getDistributionBitCount(), 8); + assertEquals(p.getDistributionBitCount(), 8); + assertEquals(iter.getBucketSource().getDistributionBitCount(), 8); + + assertEquals(p.getTotalBucketCount(), 1 << 8); + assertEquals(p.getFinishedBucketCount(), 24 << 1); + assertEquals(p.getPendingBucketCount(), 1 << 4); // Split 4 -> 7 bits, then 7 -> 8 bits + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(p.getBucketCursor(), 24*2 + (1 << 4)); + assertTrue(iter.hasNext()); + } + + public void testImportInconsistentProgressDecrease() throws ParseException { + // Bucket progress "file" that upon time of changing from 4 to 7 + // distribution bits and writing the progress had an active bucket + String input = "VDS bucket progress file\n" + + "7\n" + + "32\n" + + "24\n" + + "128\n" + + "100000000000000c:0\n"; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(input); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, 1, p); + + assertEquals(iter.getDistributionBitCount(), 7); + // Now we're at 6 distribution bits + iter.setDistributionBitCount(6); + + assertEquals(iter.getDistributionBitCount(), 6); + assertEquals(p.getDistributionBitCount(), 6); + assertEquals(iter.getBucketSource().getDistributionBitCount(), 6); + + assertEquals(p.getTotalBucketCount(), 1 << 6); + assertEquals(p.getFinishedBucketCount(), 24 >> 1); + assertEquals(p.getPendingBucketCount(), 1 << 2); // Split 4 -> 7 bits, merge 7 -> 6 bits + assertEquals(p.getActiveBucketCount(), 0); + assertEquals(p.getBucketCursor(), 24/2 + (1 << 2)); + assertTrue(iter.hasNext()); + } + + public void testEntireBucketSpaceCovered() throws ParseException { + int db = 4; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[3]; + + for (int i = 0; i < 3; ++i) { + bpp[i] = iter.getNext(); + } + for (int i = 0; i < 3; ++i) { + // Must use non-zero progress or all pending will be optimized + // away by the reset-logic + iter.update(bpp[i].getSuperbucket(), + new BucketId(db + 1, bpp[i].getSuperbucket().getId())); + } + + Set<BucketId> buckets = new TreeSet<BucketId>(); + db = 7; + for (int i = 0; i < (1 << db); ++i) { + buckets.add(new BucketId(db, i)); + } + + iter.setDistributionBitCount(db); + assertEquals(p.getFinishedBucketCount(), 0); + assertEquals(p.getPendingBucketCount(), 3 << 3); + + // Ensure all buckets are visited once and only once + while (iter.hasNext()) { + VisitorIterator.BucketProgress bp = iter.getNext(); + assertTrue(buckets.contains(bp.getSuperbucket())); + buckets.remove(bp.getSuperbucket()); + } + + assertTrue(buckets.isEmpty()); + } + + public void testExceptionOnWrongDocumentSelection() throws ParseException { + BucketIdFactory idFactory = new BucketIdFactory(); + // Since we don't store the actual original document selection in the + // progress files, we can't really tell whether or not a "wrong" document + // selection has been given, so we just do a best effort by checking + // that the number of total buckets match up and that the bucket cursor + // isn't set for explicit sources + + // Try to pass a known document selection to an unknown docsel iterator + boolean caughtIt = false; + try { + ProgressToken p = new ProgressToken("VDS bucket progress file\n16\n3\n1\n3\n" + + "8000000000001f49:0\n8000000000001a85:0\n"); + + VisitorIterator.createFromDocumentSelection("id.group != \"yahoo.com\"", idFactory, 16, p); + } + catch (IllegalArgumentException e) { + caughtIt = true; + } + assertTrue(caughtIt); + + // Now try it the other way around + caughtIt = false; + try { + ProgressToken p = new ProgressToken("VDS bucket progress file\n" + + "10\n" + + "503\n" + + "500\n" + + "1024\n" + + "28000000000000be:0\n" + + "28000000000002be:0\n" + + "28000000000001be:0\n"); + + VisitorIterator.createFromDocumentSelection("id.group=\"yahoo.com\" or id.user=555", idFactory, 16, p); + } + catch (IllegalArgumentException e) { + caughtIt = true; + } + assertTrue(caughtIt); + } + + public void testIsBucketFinished() throws ParseException { + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, 4, p); + + assertFalse(p.isBucketFinished(new BucketId(32, 0))); + // Finish superbucket 0x0000 + iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET); + assertTrue(p.isBucketFinished(new BucketId(32, 0))); + // Cursor is 1, but bucket 0x1000 not yet returned + assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3))); + VisitorIterator.BucketProgress bp = iter.getNext(); + // Cursor 2, 0x1000 returned but is contained in state, so not finished + assertFalse(p.isBucketFinished(new BucketId(32, 1 << 3))); + iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + assertTrue(p.isBucketFinished(new BucketId(32, 1 << 3))); + // Only superbucket part is used + assertTrue(p.isBucketFinished(new BucketId(32, 0x12345670))); // ...0000 + assertTrue(p.isBucketFinished(new BucketId(32, 0x12345678))); // ...1000 + assertFalse(p.isBucketFinished(new BucketId(32, 0x12345671))); // ...0001 + assertFalse(p.isBucketFinished(new BucketId(32, 0x12345679))); // ...1001 + } + + // Test that altering distribution bit count sets ProgressToken as + // inconsistent when there are active buckets + public void testInconsistentState() throws ParseException { + int db = 16; + BucketIdFactory idFactory = new BucketIdFactory(); + ProgressToken p = new ProgressToken(); + + VisitorIterator iter = VisitorIterator.createFromDocumentSelection( + "id.group != \"yahoo.com\"", idFactory, db, p); + + // For this test, have 3 finished bucket, 2 pending and 1 active + for (int i = 0; i < 3; ++i) { + iter.update(iter.getNext().getSuperbucket(), ProgressToken.FINISHED_BUCKET); + } + + VisitorIterator.BucketProgress[] bpp = new VisitorIterator.BucketProgress[2]; + bpp[0] = iter.getNext(); + bpp[1] = iter.getNext(); + VisitorIterator.BucketProgress bpa = iter.getNext(); // Leave this hanging as active + iter.update(bpp[0].getSuperbucket(), new BucketId()); + iter.update(bpp[1].getSuperbucket(), new BucketId()); + + assertFalse(p.isInconsistentState()); + iter.setDistributionBitCount(20); + assertTrue(p.isInconsistentState()); + + // Finish active, triggering the consistency fixes + iter.update(bpa.getSuperbucket(), ProgressToken.FINISHED_BUCKET); + + assertFalse(p.isInconsistentState()); + } + + public void testMalformedProgressFile() { + boolean caughtIt = false; + try { + new ProgressToken("VDS bucket progress file\n" + + "10\n" + + "503\n" + + "500\n" + + "1024\n" + + "28000000000000be:0\n" + + "28000000000002be:"); + } catch (IllegalArgumentException e) { + caughtIt = true; + } + assertTrue(caughtIt); + } + + public void testFailOnTooFewLinesInFile() { + boolean caughtIt = false; + try { + new ProgressToken("VDS bucket progress file\n" + + "10\n" + + "503\n"); + } catch (IllegalArgumentException e) { + caughtIt = true; + } + assertTrue(caughtIt); + } + + public void testUnknownFirstHeaderLine() { + boolean caughtIt = false; + try { + new ProgressToken("Smurf Time 3000\n" + + "10\n" + + "503\n" + + "500\n" + + "1024\n" + + "28000000000000be:0\n" + + "28000000000002be:0"); + } catch (IllegalArgumentException e) { + caughtIt = true; + } + assertTrue(caughtIt); + } + + public void testBinaryProgressSerialization() { + String input = "VDS bucket progress file (48.828125% completed)\n" + + "10\n" + + "503\n" + + "500\n" + + "1024\n" + + "28000000000000be:0\n" + + "28000000000002be:0\n" + + "28000000000001be:0\n"; + ProgressToken p = new ProgressToken(input); + byte[] buf = p.serialize(); + ProgressToken p2 = new ProgressToken(buf); + assertEquals(input, p2.toString()); + } + } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java index 9f4eace387c..e5db70c9967 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/ReplyMergerTestCase.java @@ -1,228 +1,228 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol;
-
-import com.yahoo.collections.Tuple2;
-import com.yahoo.messagebus.*;
-import com.yahoo.messagebus.Error;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.*;
-import static org.hamcrest.CoreMatchers.*;
-
-@SuppressWarnings("deprecation")
-public class ReplyMergerTestCase {
-
- private ReplyMerger merger;
-
- @Before
- public void setUp() {
- merger = new ReplyMerger();
- }
-
- @Test
- public void mergingGenericRepliesWithNoErrorsPicksFirstReply() {
- Reply r1 = new EmptyReply();
- Reply r2 = new EmptyReply();
- Reply r3 = new EmptyReply();
- merger.merge(0, r1);
- merger.merge(1, r2);
- merger.merge(2, r3);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
-
- assertThat(ret.first, is(0));
- assertThat(ret.second, sameInstance(r1));
- }
-
- @Test
- public void mergingSingleReplyWithOneErrorReturnsEmptyReplyWithError() {
- Reply r1 = new EmptyReply();
- Error error = new Error(1234, "oh no!");
- r1.addError(error);
- merger.merge(0, r1);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
-
- assertThat(ret.first, nullValue());
- assertThat(ret.second, not(sameInstance(r1)));
- assertThatErrorsMatch(new Error[] { error }, ret);
- }
-
- @Test
- public void mergingSingleReplyWithMultipleErrorsReturnsEmptyReplyWithAllErrors() {
- Reply r1 = new EmptyReply();
- Error errors[] = new Error[] {
- new Error(1234, "oh no!"), new Error(4567, "oh dear!"),
- };
- r1.addError(errors[0]);
- r1.addError(errors[1]);
- merger.merge(0, r1);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
-
- assertThat(ret.first, nullValue());
- assertThat(ret.second, not(sameInstance(r1)));
- assertThatErrorsMatch(errors, ret);
- }
-
- @Test
- public void mergingMultipleRepliesWithMultipleErrorsReturnsEmptyReplyWithAllErrors() {
- Reply r1 = new EmptyReply();
- Reply r2 = new EmptyReply();
- Error errors[] = new Error[] {
- new Error(1234, "oh no!"), new Error(4567, "oh dear!"), new Error(678, "omg!"),
- };
- r1.addError(errors[0]);
- r1.addError(errors[1]);
- r2.addError(errors[2]);
- merger.merge(0, r1);
- merger.merge(1, r2);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
-
- assertThat(ret.first, nullValue());
- assertThat(ret.second, not(sameInstance(r1)));
- assertThat(ret.second, not(sameInstance(r2)));
- assertThatErrorsMatch(errors, ret);
- }
-
- @Test
- public void returnIgnoredReplyWhenAllRepliesHaveOnlyIgnoredErrors() {
- Reply r1 = new EmptyReply();
- Reply r2 = new EmptyReply();
- Error errors[] = new Error[] {
- new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),
- new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh dear!"),
- new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"),
- };
- r1.addError(errors[0]);
- r1.addError(errors[1]);
- r2.addError(errors[2]);
-
- merger.merge(0, r1);
- merger.merge(1, r2);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
- assertThat(ret.first, nullValue());
- assertThat(ret.second, not(sameInstance(r1)));
- assertThat(ret.second, not(sameInstance(r2)));
- // Only first ignore error from each reply
- assertThatErrorsMatch(new Error[]{ errors[0], errors[2] }, ret);
- }
-
- @Test
- public void successfulReplyTakesPrecedenceOverIgnoredReplyWhenNoErrors() {
- Reply r1 = new EmptyReply();
- Reply r2 = new EmptyReply();
- Error errors[] = new Error[] {
- new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),
- };
- r1.addError(errors[0]);
- merger.merge(0, r1);
- merger.merge(1, r2);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
- assertThat(ret.first, is(1));
- assertThat(ret.second, sameInstance(r2));
- // Only first ignore error from each reply
- assertThatErrorsMatch(new Error[]{ }, ret);
- }
-
- @Test
- public void nonIgnoredErrorTakesPrecedence() {
- Reply r1 = new EmptyReply();
- Reply r2 = new EmptyReply();
- Error errors[] = new Error[] {
- new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"),
- new Error(DocumentProtocol.ERROR_ABORTED, "kablammo!"),
- new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"),
- };
- r1.addError(errors[0]);
- r1.addError(errors[1]);
- r2.addError(errors[2]);
-
- merger.merge(0, r1);
- merger.merge(1, r2);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
- assertThat(ret.first, nullValue());
- assertThat(ret.second, not(sameInstance(r1)));
- assertThat(ret.second, not(sameInstance(r2)));
- // All errors from replies with errors are included, not those that
- // are fully ignored.
- assertThatErrorsMatch(new Error[]{ errors[0], errors[1] }, ret);
- }
-
- @Test
- public void returnRemoveDocumentReplyWhereDocWasFound() {
- RemoveDocumentReply r1 = new RemoveDocumentReply();
- RemoveDocumentReply r2 = new RemoveDocumentReply();
- RemoveDocumentReply r3 = new RemoveDocumentReply();
- r1.setWasFound(false);
- r2.setWasFound(true);
- r3.setWasFound(false);
-
- merger.merge(0, r1);
- merger.merge(1, r2);
- merger.merge(2, r3);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
- assertThat(ret.first, is(1));
- assertThat(ret.second, sameInstance((Reply) r2));
- }
-
- @Test
- public void returnFirstRemoveDocumentReplyIfNoDocsWereFound() {
- RemoveDocumentReply r1 = new RemoveDocumentReply();
- RemoveDocumentReply r2 = new RemoveDocumentReply();
- r1.setWasFound(false);
- r2.setWasFound(false);
-
- merger.merge(0, r1);
- merger.merge(1, r2);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
- assertThat(ret.first, is(0));
- assertThat(ret.second, sameInstance((Reply)r1));
- }
-
- @Test
- public void returnUpdateDocumentReplyWhereDocWasFound() {
- UpdateDocumentReply r1 = new UpdateDocumentReply();
- UpdateDocumentReply r2 = new UpdateDocumentReply();
- UpdateDocumentReply r3 = new UpdateDocumentReply();
- r1.setWasFound(false);
- r2.setWasFound(true); // return first reply
- r3.setWasFound(true);
-
- merger.merge(0, r1);
- merger.merge(1, r2);
- merger.merge(2, r3);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
- assertThat(ret.first, is(1));
- assertThat(ret.second, sameInstance((Reply)r2));
- }
-
- @Test
- public void returnGetDocumentReplyWhereDocWasFound() {
- GetDocumentReply r1 = new GetDocumentReply(null);
- GetDocumentReply r2 = new GetDocumentReply(null);
- GetDocumentReply r3 = new GetDocumentReply(null);
- r2.setLastModified(12345L);
-
- merger.merge(0, r1);
- merger.merge(1, r2);
- merger.merge(2, r3);
- Tuple2<Integer, Reply> ret = merger.mergedReply();
- assertThat(ret.first, is(1));
- assertThat(ret.second, sameInstance((Reply)r2));
- }
-
- @Test
- public void mergingZeroRepliesReturnsDefaultEmptyReply() {
- Tuple2<Integer, Reply> ret = merger.mergedReply();
- assertThat(ret.first, nullValue());
- assertThat(ret.second, instanceOf(EmptyReply.class));
- assertThatErrorsMatch(new Error[]{}, ret);
- }
-
- private void assertThatErrorsMatch(Error[] errors, Tuple2<Integer, Reply> ret) {
- assertThat(ret.second.getNumErrors(), is(errors.length));
- for (int i = 0; i < ret.second.getNumErrors(); ++i) {
- assertThat(ret.second.getError(i).getCode(), is(errors[i].getCode()));
- assertThat(ret.second.getError(i).getMessage(), is(errors[i].getMessage()));
- }
- }
-
-}
+package com.yahoo.documentapi.messagebus.protocol; + +import com.yahoo.collections.Tuple2; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; + +@SuppressWarnings("deprecation") +public class ReplyMergerTestCase { + + private ReplyMerger merger; + + @Before + public void setUp() { + merger = new ReplyMerger(); + } + + @Test + public void mergingGenericRepliesWithNoErrorsPicksFirstReply() { + Reply r1 = new EmptyReply(); + Reply r2 = new EmptyReply(); + Reply r3 = new EmptyReply(); + merger.merge(0, r1); + merger.merge(1, r2); + merger.merge(2, r3); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + + assertThat(ret.first, is(0)); + assertThat(ret.second, sameInstance(r1)); + } + + @Test + public void mergingSingleReplyWithOneErrorReturnsEmptyReplyWithError() { + Reply r1 = new EmptyReply(); + Error error = new Error(1234, "oh no!"); + r1.addError(error); + merger.merge(0, r1); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + + assertThat(ret.first, nullValue()); + assertThat(ret.second, not(sameInstance(r1))); + assertThatErrorsMatch(new Error[] { error }, ret); + } + + @Test + public void mergingSingleReplyWithMultipleErrorsReturnsEmptyReplyWithAllErrors() { + Reply r1 = new EmptyReply(); + Error errors[] = new Error[] { + new Error(1234, "oh no!"), new Error(4567, "oh dear!"), + }; + r1.addError(errors[0]); + r1.addError(errors[1]); + merger.merge(0, r1); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + + assertThat(ret.first, nullValue()); + assertThat(ret.second, not(sameInstance(r1))); + assertThatErrorsMatch(errors, ret); + } + + @Test + public void mergingMultipleRepliesWithMultipleErrorsReturnsEmptyReplyWithAllErrors() { + Reply r1 = new EmptyReply(); + Reply r2 = new EmptyReply(); + Error errors[] = new Error[] { + new Error(1234, "oh no!"), new Error(4567, "oh dear!"), new Error(678, "omg!"), + }; + r1.addError(errors[0]); + r1.addError(errors[1]); + r2.addError(errors[2]); + merger.merge(0, r1); + merger.merge(1, r2); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + + assertThat(ret.first, nullValue()); + assertThat(ret.second, not(sameInstance(r1))); + assertThat(ret.second, not(sameInstance(r2))); + assertThatErrorsMatch(errors, ret); + } + + @Test + public void returnIgnoredReplyWhenAllRepliesHaveOnlyIgnoredErrors() { + Reply r1 = new EmptyReply(); + Reply r2 = new EmptyReply(); + Error errors[] = new Error[] { + new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"), + new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh dear!"), + new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"), + }; + r1.addError(errors[0]); + r1.addError(errors[1]); + r2.addError(errors[2]); + + merger.merge(0, r1); + merger.merge(1, r2); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + assertThat(ret.first, nullValue()); + assertThat(ret.second, not(sameInstance(r1))); + assertThat(ret.second, not(sameInstance(r2))); + // Only first ignore error from each reply + assertThatErrorsMatch(new Error[]{ errors[0], errors[2] }, ret); + } + + @Test + public void successfulReplyTakesPrecedenceOverIgnoredReplyWhenNoErrors() { + Reply r1 = new EmptyReply(); + Reply r2 = new EmptyReply(); + Error errors[] = new Error[] { + new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"), + }; + r1.addError(errors[0]); + merger.merge(0, r1); + merger.merge(1, r2); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + assertThat(ret.first, is(1)); + assertThat(ret.second, sameInstance(r2)); + // Only first ignore error from each reply + assertThatErrorsMatch(new Error[]{ }, ret); + } + + @Test + public void nonIgnoredErrorTakesPrecedence() { + Reply r1 = new EmptyReply(); + Reply r2 = new EmptyReply(); + Error errors[] = new Error[] { + new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "oh no!"), + new Error(DocumentProtocol.ERROR_ABORTED, "kablammo!"), + new Error(DocumentProtocol.ERROR_MESSAGE_IGNORED, "omg!"), + }; + r1.addError(errors[0]); + r1.addError(errors[1]); + r2.addError(errors[2]); + + merger.merge(0, r1); + merger.merge(1, r2); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + assertThat(ret.first, nullValue()); + assertThat(ret.second, not(sameInstance(r1))); + assertThat(ret.second, not(sameInstance(r2))); + // All errors from replies with errors are included, not those that + // are fully ignored. + assertThatErrorsMatch(new Error[]{ errors[0], errors[1] }, ret); + } + + @Test + public void returnRemoveDocumentReplyWhereDocWasFound() { + RemoveDocumentReply r1 = new RemoveDocumentReply(); + RemoveDocumentReply r2 = new RemoveDocumentReply(); + RemoveDocumentReply r3 = new RemoveDocumentReply(); + r1.setWasFound(false); + r2.setWasFound(true); + r3.setWasFound(false); + + merger.merge(0, r1); + merger.merge(1, r2); + merger.merge(2, r3); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + assertThat(ret.first, is(1)); + assertThat(ret.second, sameInstance((Reply) r2)); + } + + @Test + public void returnFirstRemoveDocumentReplyIfNoDocsWereFound() { + RemoveDocumentReply r1 = new RemoveDocumentReply(); + RemoveDocumentReply r2 = new RemoveDocumentReply(); + r1.setWasFound(false); + r2.setWasFound(false); + + merger.merge(0, r1); + merger.merge(1, r2); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + assertThat(ret.first, is(0)); + assertThat(ret.second, sameInstance((Reply)r1)); + } + + @Test + public void returnUpdateDocumentReplyWhereDocWasFound() { + UpdateDocumentReply r1 = new UpdateDocumentReply(); + UpdateDocumentReply r2 = new UpdateDocumentReply(); + UpdateDocumentReply r3 = new UpdateDocumentReply(); + r1.setWasFound(false); + r2.setWasFound(true); // return first reply + r3.setWasFound(true); + + merger.merge(0, r1); + merger.merge(1, r2); + merger.merge(2, r3); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + assertThat(ret.first, is(1)); + assertThat(ret.second, sameInstance((Reply)r2)); + } + + @Test + public void returnGetDocumentReplyWhereDocWasFound() { + GetDocumentReply r1 = new GetDocumentReply(null); + GetDocumentReply r2 = new GetDocumentReply(null); + GetDocumentReply r3 = new GetDocumentReply(null); + r2.setLastModified(12345L); + + merger.merge(0, r1); + merger.merge(1, r2); + merger.merge(2, r3); + Tuple2<Integer, Reply> ret = merger.mergedReply(); + assertThat(ret.first, is(1)); + assertThat(ret.second, sameInstance((Reply)r2)); + } + + @Test + public void mergingZeroRepliesReturnsDefaultEmptyReply() { + Tuple2<Integer, Reply> ret = merger.mergedReply(); + assertThat(ret.first, nullValue()); + assertThat(ret.second, instanceOf(EmptyReply.class)); + assertThatErrorsMatch(new Error[]{}, ret); + } + + private void assertThatErrorsMatch(Error[] errors, Tuple2<Integer, Reply> ret) { + assertThat(ret.second.getNumErrors(), is(errors.length)); + for (int i = 0; i < ret.second.getNumErrors(); ++i) { + assertThat(ret.second.getError(i).getCode(), is(errors[i].getCode())); + assertThat(ret.second.getError(i).getMessage(), is(errors[i].getMessage())); + } + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java index 226f96f6553..9faed67ba8d 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java @@ -1,975 +1,975 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.yahoo.component.Version;
-import com.yahoo.document.*;
-import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
-import com.yahoo.document.idstring.IdString;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.documentapi.messagebus.protocol.*;
-import com.yahoo.messagebus.Routable;
-import com.yahoo.text.Utf8;
-import com.yahoo.vdslib.DocumentList;
-import com.yahoo.vdslib.Entry;
-import com.yahoo.vdslib.SearchResult;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class Messages50TestCase extends MessagesTestBase {
-
- @Override
- protected void registerTests(Map<Integer, RunnableTest> out) {
- // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
- // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
- out.put(DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateMessage());
- out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());
- out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage());
- out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage());
- out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage());
- out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage());
- out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage());
- out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage());
- out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage());
- out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage());
- out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage());
- out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage());
- out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage());
- out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage());
- out.put(DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateReply());
- out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply());
- out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply());
- out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply());
- out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply());
- out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply());
- out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply());
- out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply());
- out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply());
- out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply());
- out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply());
- out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply());
- out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply());
- out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply());
- out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply());
- out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply());
- out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply());
- out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply());
- out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply());
- }
-
- @Override
- protected Version version() {
- return new Version(5, 0);
- }
-
- @Override
- protected boolean shouldTestCoverage() {
- return false;
- }
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Tests
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- private static int BASE_MESSAGE_LENGTH = 5;
-
- public class testRemoveLocationMessage implements RunnableTest {
-
- @Override
- public void run() {
- {
- RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\"");
- assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang);
- assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection());
- }
- }
- }
- }
-
- public class testGetBucketListMessage implements RunnableTest {
-
- @Override
- public void run() {
- GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123));
- msg.setLoadType(loadTypes.getNameMap().get("foo"));
- assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang);
- assertEquals(new BucketId(16, 123), msg.getBucketId());
- assertEquals("foo", msg.getLoadType().getName());
- }
- }
- }
-
-
- public class testStatBucketMessage implements RunnableTest {
-
- @Override
- public void run() {
- StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123");
- msg.setLoadType(null);
- assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang);
- assertEquals(new BucketId(16, 123), msg.getBucketId());
- assertEquals("id.user=123", msg.getDocumentSelection());
- assertEquals("default", msg.getLoadType().getName());
- }
- }
- }
-
- public class testGetBucketStateMessage implements RunnableTest {
-
- @Override
- public void run() {
- GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666));
- assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang);
- assertEquals(16, msg.getBucketId().getUsedBits());
- assertEquals(4611686018427388570l, msg.getBucketId().getId());
- }
- }
- }
-
- public class testCreateVisitorMessage implements RunnableTest {
-
- @Override
- @SuppressWarnings("deprecation")
- public void run() {
- CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");
- msg.setDocumentSelection("true and false or true");
- msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));
- msg.getParameters().put("anothervar", Utf8.toBytes("34"));
- msg.getBuckets().add(new BucketId(16, 1234));
- msg.setVisitRemoves(true);
- msg.setVisitorOrdering(OrderingSpecification.DESCENDING);
- msg.setMaxBucketsPerVisitor(2);
- assertEquals(BASE_MESSAGE_LENGTH + 168, serialize("CreateVisitorMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);
- assertEquals("SomeLibrary", msg.getLibraryName());
- assertEquals("myvisitor", msg.getInstanceId());
- assertEquals("newyork", msg.getControlDestination());
- assertEquals("london", msg.getDataDestination());
- assertEquals("true and false or true", msg.getDocumentSelection());
- assertEquals(8, msg.getMaxPendingReplyCount());
- assertEquals(true, msg.getVisitRemoves());
- assertEquals(false, msg.getVisitInconsistentBuckets());
- assertEquals(1, msg.getBuckets().size());
- assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());
- assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));
- assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));
- assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());
- assertEquals(2, msg.getMaxBucketsPerVisitor());
- }
-
- msg.getBuckets().clear();
-
- assertEquals("CreateVisitorMessage(" +
- "No buckets, " +
- "selection 'true and false or true', " +
- "library SomeLibrary, including removes, " +
- "get fields: [all]" +
- ")",
- msg.toString());
-
- msg.getBuckets().add(new BucketId(16, 1234));
-
- assertEquals("CreateVisitorMessage(" +
- "Bucket BucketId(0x40000000000004d2), " +
- "selection 'true and false or true', " +
- "library SomeLibrary, including removes, " +
- "get fields: [all]" +
- ")",
- msg.toString());
-
- msg.getBuckets().add(new BucketId(16, 1235));
- msg.getBuckets().add(new BucketId(16, 1236));
- msg.getBuckets().add(new BucketId(16, 1237));
- msg.getBuckets().add(new BucketId(16, 1238));
- msg.setFromTimestamp(10001);
- msg.setToTimestamp(20002);
- msg.setVisitInconsistentBuckets(true);
- assertEquals("CreateVisitorMessage(" +
- "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " +
- "time 10001-20002, " +
- "selection 'true and false or true', " +
- "library SomeLibrary, including removes, " +
- "get fields: [all], " +
- "visit inconsistent buckets" +
- ")",
- msg.toString());
- }
- }
-
- public class testCreateVisitorReply implements RunnableTest {
-
- @Override
- public void run() {
- CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR);
- reply.setLastBucket(new BucketId(16, 123));
- reply.getVisitorStatistics().setBucketsVisited(3);
- reply.getVisitorStatistics().setDocumentsVisited(1000);
- reply.getVisitorStatistics().setBytesVisited(1024000);
- reply.getVisitorStatistics().setDocumentsReturned(123);
- reply.getVisitorStatistics().setBytesReturned(512000);
- reply.getVisitorStatistics().setSecondPassDocumentsReturned(456);
- reply.getVisitorStatistics().setSecondPassBytesReturned(789100);
-
- assertEquals(65, serialize("CreateVisitorReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang);
- assertNotNull(reply);
- assertEquals(new BucketId(16, 123), reply.getLastBucket());
- assertEquals(3, reply.getVisitorStatistics().getBucketsVisited());
- assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited());
- assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited());
- assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned());
- assertEquals(512000, reply.getVisitorStatistics().getBytesReturned());
- assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned());
- assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned());
- }
- }
- }
-
- public class testDestroyVisitorReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR);
- }
- }
-
- public class testDocumentListReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST);
- }
- }
-
- public class testDocumentSummaryReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY);
- }
- }
-
- public class testEmptyBucketsReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS);
- }
- }
-
- public class testDestroyVisitorMessage implements RunnableTest {
-
- @Override
- public void run() {
- DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor");
- assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang);
- assertEquals("myvisitor", msg.getInstanceId());
- }
- }
- }
-
- public class testDocumentListMessage implements RunnableTest {
-
- @Override
- public void run() {
- DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP);
- assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
- assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
- assertFalse(msg.getDocuments().get(0).isRemoveEntry());
-
- assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg));
- msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA);
- assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString());
- assertEquals(1234, msg.getDocuments().get(0).getTimestamp());
- assertFalse(msg.getDocuments().get(0).isRemoveEntry());
-
- }
- }
-
- public class testEmptyBucketsMessage implements RunnableTest {
-
- @Override
- public void run() {
- List<BucketId> bids = new ArrayList<>();
- for (int i = 0; i < 13; ++i) {
- bids.add(new BucketId(16, i));
- }
-
- EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids);
- assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm));
- for (Language lang : LANGUAGES) {
- ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang);
- for (int i = 0; i < 13; ++i) {
- assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i));
- }
- }
- }
- }
-
- public class testDocumentSummaryMessage implements RunnableTest {
-
- @Override
- public void run() {
- try {
- FileInputStream stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-1.dat"));
- byte[] data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- Routable routable = decode(data);
- assertTrue(routable instanceof DocumentSummaryMessage);
-
- DocumentSummaryMessage msg = (DocumentSummaryMessage)routable;
- assertEquals(0, msg.getResult().getSummaryCount());
-
- stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-2.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof DocumentSummaryMessage);
-
- msg = (DocumentSummaryMessage)routable;
- assertEquals(2, msg.getResult().getSummaryCount());
- com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0);
- assertEquals("doc1", s.getDocId());
- byte[] b = s.getSummary();
- assertEquals(8, b.length);
- byte[] c = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(c[i], b[i]);
- }
-
- s = msg.getResult().getSummary(1);
- assertEquals("aoc17", s.getDocId());
- b = s.getSummary();
- assertEquals(9, b.length);
- byte[] d = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(d[i], b[i]);
- }
-
- stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-3.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof DocumentSummaryMessage);
-
- msg = (DocumentSummaryMessage)routable;
- assertEquals(2, msg.getResult().getSummaryCount());
-
- s = msg.getResult().getSummary(0);
- assertEquals("aoc17", s.getDocId());
- b = s.getSummary();
- assertEquals(9, b.length);
- byte[] e = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(e[i], b[i]);
- }
-
- s = msg.getResult().getSummary(1);
- assertEquals("doc1", s.getDocId());
- b = s.getSummary();
- assertEquals(8, b.length);
- byte[] f = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(f[i], b[i]);
- }
- } catch (IOException e) {
- fail(e.toString());
- }
- }
- }
-
-
- public class testGetDocumentMessage implements RunnableTest {
-
- @Override
- @SuppressWarnings("deprecation")
- public void run() {
- GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"));
- assertEquals(BASE_MESSAGE_LENGTH + 20, serialize("GetDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);
- assertEquals("doc:scheme:", msg.getDocumentId().toString());
- }
- }
- }
-
-
- public class testRemoveDocumentMessage implements RunnableTest {
-
- @Override
- public void run() {
- RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
- assertEquals(BASE_MESSAGE_LENGTH + 16, serialize("RemoveDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);
- assertEquals("doc:scheme:", msg.getDocumentId().toString());
- }
- }
- }
-
- public class testMapVisitorMessage implements RunnableTest {
-
- @Override
- public void run() {
- MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP);
- assertEquals("3", msg.getData().get("foo"));
- assertEquals("5", msg.getData().get("bar"));
-
- assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg));
-
- msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA);
- assertEquals("3", msg.getData().get("foo"));
- assertEquals("5", msg.getData().get("bar"));
- }
- }
-
-
- public class testVisitorInfoMessage implements RunnableTest {
-
- @Override
- public void run() {
- VisitorInfoMessage msg = new VisitorInfoMessage();
- msg.getFinishedBuckets().add(new BucketId(16, 1));
- msg.getFinishedBuckets().add(new BucketId(16, 2));
- msg.getFinishedBuckets().add(new BucketId(16, 4));
- msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6");
- assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang);
- assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1)));
- assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2)));
- assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4)));
- assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage());
- }
- }
- }
-
- public class testSearchResultMessage implements RunnableTest {
-
- @Override
- public void run() throws Exception {
- FileInputStream stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-1.dat"));
- byte[] data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- Routable routable = decode(data);
- assertTrue(routable instanceof SearchResultMessage);
-
- SearchResultMessage msg = (SearchResultMessage)routable;
- assertEquals(0, msg.getResult().getHitCount());
-
- stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-2.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof SearchResultMessage);
-
- msg = (SearchResultMessage)routable;
- assertEquals(2, msg.getResult().getHitCount());
- com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
- h = msg.getResult().getHit(1);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
-
- stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-3.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof SearchResultMessage);
-
- msg = (SearchResultMessage)routable;
- assertEquals(2, msg.getResult().getHitCount());
- h = msg.getResult().getHit(0);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
- h = msg.getResult().getHit(1);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
-
- stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-4.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof SearchResultMessage);
-
- msg = (SearchResultMessage)routable;
- assertEquals(3, msg.getResult().getHitCount());
- h = msg.getResult().getHit(0);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
- byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(e[i], b[i]);
- }
- h = msg.getResult().getHit(1);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
- b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(d[i], b[i]);
- }
- h = msg.getResult().getHit(2);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(90.0, h.getRank(), 1E-6);
- assertEquals("doc18", h.getDocId());
- b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(c[i], b[i]);
- }
- }
- }
-
- public class testPutDocumentMessage implements RunnableTest {
-
- @Override
- public void run() {
- PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));
- msg.setTimestamp(666);
- assertEquals(BASE_MESSAGE_LENGTH + 41, serialize("PutDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
- assertEquals("testdoc", msg.getDocumentPut().getDocument().getDataType().getName());
- assertEquals("doc:scheme:", msg.getDocumentPut().getDocument().getId().toString());
- assertEquals(666, msg.getTimestamp());
- }
- }
- }
-
- public class testPutDocumentReply implements RunnableTest {
-
- @Override
- public void run() {
- WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT);
- reply.setHighestModificationTimestamp(30);
-
- assertEquals(13, serialize("PutDocumentReply", reply));
-
- for (Language lang : LANGUAGES) {
- WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang);
- assertNotNull(obj);
- assertEquals(30, obj.getHighestModificationTimestamp());
- }
- }
- }
-
- public class testUpdateDocumentMessage implements RunnableTest {
-
- @Override
- public void run() {
- DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
- DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:"));
- update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
- UpdateDocumentMessage msg = new UpdateDocumentMessage(update);
- msg.setNewTimestamp(777);
- msg.setOldTimestamp(666);
-
- assertEquals(BASE_MESSAGE_LENGTH + 89, serialize("UpdateDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (UpdateDocumentMessage)deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang);
- assertEquals(update, msg.getDocumentUpdate());
- assertEquals(777, msg.getNewTimestamp());
- assertEquals(666, msg.getOldTimestamp());
- }
- }
- }
-
- public class testUpdateDocumentReply implements RunnableTest {
-
- @Override
- public void run() {
- UpdateDocumentReply reply = new UpdateDocumentReply();
- reply.setHighestModificationTimestamp(30);
- reply.setWasFound(false);
-
- assertEquals(14, serialize("UpdateDocumentReply", reply));
-
- for (Language lang : LANGUAGES) {
- UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang);
- assertNotNull(obj);
- assertEquals(30, reply.getHighestModificationTimestamp());
- assertEquals(false, obj.wasFound());
- }
- }
- }
-
- public class testVisitorInfoReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO);
- }
- }
-
- public class testWrongDistributionReply implements RunnableTest {
-
- @Override
- public void run() {
- WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2");
- assertEquals(32, serialize("WrongDistributionReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang);
- assertEquals("distributor:3 storage:2", reply.getSystemState());
- }
- }
- }
-
- public class testRemoveDocumentReply implements RunnableTest {
-
- @Override
- public void run() {
- RemoveDocumentReply reply = new RemoveDocumentReply();
- reply.setHighestModificationTimestamp(30);
- reply.setWasFound(false);
-
- assertEquals(14, serialize("RemoveDocumentReply", reply));
-
- for (Language lang : LANGUAGES) {
- RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang);
- assertNotNull(obj);
- assertEquals(30, obj.getHighestModificationTimestamp());
- assertEquals(false, obj.wasFound());
- }
- }
- }
-
- public class testRemoveLocationReply implements RunnableTest {
-
- @Override
- public void run() {
- testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION);
- }
- }
-
- public class testSearchResultReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT);
- }
- }
-
- public class testStatBucketReply implements RunnableTest {
-
- @Override
- public void run() {
- StatBucketReply msg = new StatBucketReply();
- msg.setResults("These are the votes of the Norwegian jury");
-
- assertEquals(50, serialize("StatBucketReply", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang);
- assertEquals("These are the votes of the Norwegian jury", msg.getResults());
- }
- }
- }
-
- public class testBatchDocumentUpdateMessage implements RunnableTest {
-
- @Override
- public void run() {
- DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
- BatchDocumentUpdateMessage msg = new BatchDocumentUpdateMessage(1234);
-
- {
- DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("userdoc:footype:1234:foo"));
- update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
- msg.addUpdate(update);
- }
- {
- DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc(32,17):footype:1234:123456789:foo"));
- update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
- msg.addUpdate(update);
- }
-
- try {
- DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc:footype:5678:foo"));
- update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
- msg.addUpdate(update);
- fail();
- } catch (Exception e) {
-
- }
-
- try {
- DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("groupdoc:footype:hable:foo"));
- update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
- msg.addUpdate(update);
- fail();
- } catch (Exception e) {
-
- }
-
- assertEquals(2, msg.getUpdates().size());
-
- assertEquals(BASE_MESSAGE_LENGTH + 202, serialize("BatchDocumentUpdateMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (BatchDocumentUpdateMessage)deserialize("BatchDocumentUpdateMessage", DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, lang);
- assertEquals(2, msg.getUpdates().size());
- }
- }
- }
-
- public class testBatchDocumentUpdateReply implements RunnableTest {
-
- @Override
- public void run() {
- BatchDocumentUpdateReply reply = new BatchDocumentUpdateReply();
- reply.setHighestModificationTimestamp(30);
- reply.getDocumentsNotFound().add(false);
- reply.getDocumentsNotFound().add(true);
- reply.getDocumentsNotFound().add(true);
-
- assertEquals(20, serialize("BatchDocumentUpdateReply", reply));
-
- for (Language lang : LANGUAGES) {
- BatchDocumentUpdateReply obj = (BatchDocumentUpdateReply)deserialize("BatchDocumentUpdateReply", DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, lang);
- assertNotNull(obj);
- assertEquals(30, obj.getHighestModificationTimestamp());
- assertEquals(3, obj.getDocumentsNotFound().size());
- assertFalse(obj.getDocumentsNotFound().get(0));
- assertTrue(obj.getDocumentsNotFound().get(1));
- assertTrue(obj.getDocumentsNotFound().get(2));
- }
- }
- }
-
-
- public class testQueryResultReply implements RunnableTest {
-
- @Override
- public void run() {
- testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT);
- }
- }
-
- public class testQueryResultMessage implements RunnableTest {
-
- @Override
- public void run() throws Exception {
- FileInputStream stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-1.dat"));
- byte[] data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- Routable routable = decode(data);
- assertTrue(routable instanceof QueryResultMessage);
-
- QueryResultMessage msg = (QueryResultMessage)routable;
- assertEquals(0, msg.getResult().getHitCount());
-
- stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-2.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof QueryResultMessage);
-
- msg = (QueryResultMessage)routable;
- assertEquals(2, msg.getResult().getHitCount());
- com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
- h = msg.getResult().getHit(1);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
-
- stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-3.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof QueryResultMessage);
-
- msg = (QueryResultMessage)routable;
- assertEquals(2, msg.getResult().getHitCount());
- h = msg.getResult().getHit(0);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
- h = msg.getResult().getHit(1);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
-
- stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-4.dat"));
- data = new byte[stream.available()];
- assertEquals(data.length, stream.read(data));
-
- routable = decode(data);
- assertTrue(routable instanceof QueryResultMessage);
-
- msg = (QueryResultMessage)routable;
- assertEquals(3, msg.getResult().getHitCount());
- h = msg.getResult().getHit(0);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(89.0, h.getRank(), 1E-6);
- assertEquals("doc1", h.getDocId());
- byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(e[i], b[i]);
- }
- h = msg.getResult().getHit(1);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(109.0, h.getRank(), 1E-6);
- assertEquals("doc17", h.getDocId());
- b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(d[i], b[i]);
- }
- h = msg.getResult().getHit(2);
- assertTrue(h instanceof SearchResult.HitWithSortBlob);
- assertEquals(90.0, h.getRank(), 1E-6);
- assertEquals("doc18", h.getDocId());
- b = ((SearchResult.HitWithSortBlob)h).getSortBlob();
- assertEquals(9, b.length);
- byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' };
- for (int i = 0; i < b.length; i++) {
- assertEquals(c[i], b[i]);
- }
- }
- }
-
- public class testGetBucketListReply implements RunnableTest {
-
- public void run() {
- GetBucketListReply reply = new GetBucketListReply();
- reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
- reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
- reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
-
- assertEquals(56, serialize("GetBucketListReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang);
- assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo"));
- assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar"));
- assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink"));
- }
- }
- }
-
- public class testGetBucketStateReply implements RunnableTest {
-
- public void run() {
- GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo"));
- GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar"));
-
- GetBucketStateReply reply = new GetBucketStateReply();
- List<DocumentState> state = new ArrayList<>(2);
- state.add(new DocumentState(foo, 777, false));
- state.add(new DocumentState(bar, 888, true));
- reply.setBucketState(state);
- assertEquals(53, serialize("GetBucketStateReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang);
- assertEquals(777, reply.getBucketState().get(0).getTimestamp());
- assertEquals(foo, reply.getBucketState().get(0).getGid());
- assertEquals(false, reply.getBucketState().get(0).isRemoveEntry());
- assertEquals(888, reply.getBucketState().get(1).getTimestamp());
- assertEquals(bar, reply.getBucketState().get(1).getGid());
- assertEquals(true, reply.getBucketState().get(1).isRemoveEntry());
- }
- }
- }
-
- public class testGetDocumentReply implements RunnableTest {
-
- public void run() {
- GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"));
- assertEquals(43, serialize("GetDocumentReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang);
- assertEquals("testdoc", reply.getDocument().getDataType().getName());
- assertEquals("doc:scheme:", reply.getDocument().getId().toString());
- }
- }
- }
-
- public class testMapVisitorReply implements RunnableTest {
-
- public void run() {
- testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR);
- }
- }
-
- protected void testDocumentReply(String filename, int type) {
- DocumentReply reply = new DocumentReply(type);
- assertEquals(5, serialize(filename, reply));
-
- for (Language lang : LANGUAGES) {
- reply = (DocumentReply)deserialize(filename, type, lang);
- assertNotNull(reply);
- }
- }
-
- protected void testVisitorReply(String filename, int type) {
- VisitorReply reply = new VisitorReply(type);
- assertEquals(5, serialize(filename, reply));
-
- for (Language lang : LANGUAGES) {
- reply = (VisitorReply)deserialize(filename, type, lang);
- assertNotNull(reply);
- }
- }
-
-}
+package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.component.Version; +import com.yahoo.document.*; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; +import com.yahoo.document.idstring.IdString; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.messagebus.Routable; +import com.yahoo.text.Utf8; +import com.yahoo.vdslib.DocumentList; +import com.yahoo.vdslib.Entry; +import com.yahoo.vdslib.SearchResult; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Messages50TestCase extends MessagesTestBase { + + @Override + protected void registerTests(Map<Integer, RunnableTest> out) { + // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support + // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + out.put(DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateMessage()); + out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage()); + out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage()); + out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage()); + out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage()); + out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage()); + out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage()); + out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage()); + out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage()); + out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage()); + out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage()); + out.put(DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, new testBatchDocumentUpdateReply()); + out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply()); + out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply()); + out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply()); + out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply()); + out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply()); + out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply()); + out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply()); + out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply()); + out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply()); + out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply()); + out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply()); + out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply()); + out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply()); + out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply()); + out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply()); + out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply()); + out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply()); + out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply()); + } + + @Override + protected Version version() { + return new Version(5, 0); + } + + @Override + protected boolean shouldTestCoverage() { + return false; + } + + //////////////////////////////////////////////////////////////////////////////// + // + // Tests + // + //////////////////////////////////////////////////////////////////////////////// + + private static int BASE_MESSAGE_LENGTH = 5; + + public class testRemoveLocationMessage implements RunnableTest { + + @Override + public void run() { + { + RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\""); + assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang); + assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection()); + } + } + } + } + + public class testGetBucketListMessage implements RunnableTest { + + @Override + public void run() { + GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); + msg.setLoadType(loadTypes.getNameMap().get("foo")); + assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); + assertEquals(new BucketId(16, 123), msg.getBucketId()); + assertEquals("foo", msg.getLoadType().getName()); + } + } + } + + + public class testStatBucketMessage implements RunnableTest { + + @Override + public void run() { + StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123"); + msg.setLoadType(null); + assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang); + assertEquals(new BucketId(16, 123), msg.getBucketId()); + assertEquals("id.user=123", msg.getDocumentSelection()); + assertEquals("default", msg.getLoadType().getName()); + } + } + } + + public class testGetBucketStateMessage implements RunnableTest { + + @Override + public void run() { + GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666)); + assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang); + assertEquals(16, msg.getBucketId().getUsedBits()); + assertEquals(4611686018427388570l, msg.getBucketId().getId()); + } + } + } + + public class testCreateVisitorMessage implements RunnableTest { + + @Override + @SuppressWarnings("deprecation") + public void run() { + CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london"); + msg.setDocumentSelection("true and false or true"); + msg.getParameters().put("myvar", Utf8.toBytes("somevalue")); + msg.getParameters().put("anothervar", Utf8.toBytes("34")); + msg.getBuckets().add(new BucketId(16, 1234)); + msg.setVisitRemoves(true); + msg.setVisitorOrdering(OrderingSpecification.DESCENDING); + msg.setMaxBucketsPerVisitor(2); + assertEquals(BASE_MESSAGE_LENGTH + 168, serialize("CreateVisitorMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang); + assertEquals("SomeLibrary", msg.getLibraryName()); + assertEquals("myvisitor", msg.getInstanceId()); + assertEquals("newyork", msg.getControlDestination()); + assertEquals("london", msg.getDataDestination()); + assertEquals("true and false or true", msg.getDocumentSelection()); + assertEquals(8, msg.getMaxPendingReplyCount()); + assertEquals(true, msg.getVisitRemoves()); + assertEquals(false, msg.getVisitInconsistentBuckets()); + assertEquals(1, msg.getBuckets().size()); + assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next()); + assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar"))); + assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar"))); + assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering()); + assertEquals(2, msg.getMaxBucketsPerVisitor()); + } + + msg.getBuckets().clear(); + + assertEquals("CreateVisitorMessage(" + + "No buckets, " + + "selection 'true and false or true', " + + "library SomeLibrary, including removes, " + + "get fields: [all]" + + ")", + msg.toString()); + + msg.getBuckets().add(new BucketId(16, 1234)); + + assertEquals("CreateVisitorMessage(" + + "Bucket BucketId(0x40000000000004d2), " + + "selection 'true and false or true', " + + "library SomeLibrary, including removes, " + + "get fields: [all]" + + ")", + msg.toString()); + + msg.getBuckets().add(new BucketId(16, 1235)); + msg.getBuckets().add(new BucketId(16, 1236)); + msg.getBuckets().add(new BucketId(16, 1237)); + msg.getBuckets().add(new BucketId(16, 1238)); + msg.setFromTimestamp(10001); + msg.setToTimestamp(20002); + msg.setVisitInconsistentBuckets(true); + assertEquals("CreateVisitorMessage(" + + "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " + + "time 10001-20002, " + + "selection 'true and false or true', " + + "library SomeLibrary, including removes, " + + "get fields: [all], " + + "visit inconsistent buckets" + + ")", + msg.toString()); + } + } + + public class testCreateVisitorReply implements RunnableTest { + + @Override + public void run() { + CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); + reply.setLastBucket(new BucketId(16, 123)); + reply.getVisitorStatistics().setBucketsVisited(3); + reply.getVisitorStatistics().setDocumentsVisited(1000); + reply.getVisitorStatistics().setBytesVisited(1024000); + reply.getVisitorStatistics().setDocumentsReturned(123); + reply.getVisitorStatistics().setBytesReturned(512000); + reply.getVisitorStatistics().setSecondPassDocumentsReturned(456); + reply.getVisitorStatistics().setSecondPassBytesReturned(789100); + + assertEquals(65, serialize("CreateVisitorReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang); + assertNotNull(reply); + assertEquals(new BucketId(16, 123), reply.getLastBucket()); + assertEquals(3, reply.getVisitorStatistics().getBucketsVisited()); + assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited()); + assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited()); + assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned()); + assertEquals(512000, reply.getVisitorStatistics().getBytesReturned()); + assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); + assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned()); + } + } + } + + public class testDestroyVisitorReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR); + } + } + + public class testDocumentListReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST); + } + } + + public class testDocumentSummaryReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY); + } + } + + public class testEmptyBucketsReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS); + } + } + + public class testDestroyVisitorMessage implements RunnableTest { + + @Override + public void run() { + DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor"); + assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang); + assertEquals("myvisitor", msg.getInstanceId()); + } + } + } + + public class testDocumentListMessage implements RunnableTest { + + @Override + public void run() { + DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP); + assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); + assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); + assertFalse(msg.getDocuments().get(0).isRemoveEntry()); + + assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg)); + msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA); + assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); + assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); + assertFalse(msg.getDocuments().get(0).isRemoveEntry()); + + } + } + + public class testEmptyBucketsMessage implements RunnableTest { + + @Override + public void run() { + List<BucketId> bids = new ArrayList<>(); + for (int i = 0; i < 13; ++i) { + bids.add(new BucketId(16, i)); + } + + EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids); + assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm)); + for (Language lang : LANGUAGES) { + ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang); + for (int i = 0; i < 13; ++i) { + assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i)); + } + } + } + } + + public class testDocumentSummaryMessage implements RunnableTest { + + @Override + public void run() { + try { + FileInputStream stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-1.dat")); + byte[] data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + Routable routable = decode(data); + assertTrue(routable instanceof DocumentSummaryMessage); + + DocumentSummaryMessage msg = (DocumentSummaryMessage)routable; + assertEquals(0, msg.getResult().getSummaryCount()); + + stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-2.dat")); + data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + routable = decode(data); + assertTrue(routable instanceof DocumentSummaryMessage); + + msg = (DocumentSummaryMessage)routable; + assertEquals(2, msg.getResult().getSummaryCount()); + com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0); + assertEquals("doc1", s.getDocId()); + byte[] b = s.getSummary(); + assertEquals(8, b.length); + byte[] c = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' }; + for (int i = 0; i < b.length; i++) { + assertEquals(c[i], b[i]); + } + + s = msg.getResult().getSummary(1); + assertEquals("aoc17", s.getDocId()); + b = s.getSummary(); + assertEquals(9, b.length); + byte[] d = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' }; + for (int i = 0; i < b.length; i++) { + assertEquals(d[i], b[i]); + } + + stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-3.dat")); + data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + routable = decode(data); + assertTrue(routable instanceof DocumentSummaryMessage); + + msg = (DocumentSummaryMessage)routable; + assertEquals(2, msg.getResult().getSummaryCount()); + + s = msg.getResult().getSummary(0); + assertEquals("aoc17", s.getDocId()); + b = s.getSummary(); + assertEquals(9, b.length); + byte[] e = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' }; + for (int i = 0; i < b.length; i++) { + assertEquals(e[i], b[i]); + } + + s = msg.getResult().getSummary(1); + assertEquals("doc1", s.getDocId()); + b = s.getSummary(); + assertEquals(8, b.length); + byte[] f = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' }; + for (int i = 0; i < b.length; i++) { + assertEquals(f[i], b[i]); + } + } catch (IOException e) { + fail(e.toString()); + } + } + } + + + public class testGetDocumentMessage implements RunnableTest { + + @Override + @SuppressWarnings("deprecation") + public void run() { + GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:")); + assertEquals(BASE_MESSAGE_LENGTH + 20, serialize("GetDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang); + assertEquals("doc:scheme:", msg.getDocumentId().toString()); + } + } + } + + + public class testRemoveDocumentMessage implements RunnableTest { + + @Override + public void run() { + RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); + assertEquals(BASE_MESSAGE_LENGTH + 16, serialize("RemoveDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang); + assertEquals("doc:scheme:", msg.getDocumentId().toString()); + } + } + } + + public class testMapVisitorMessage implements RunnableTest { + + @Override + public void run() { + MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP); + assertEquals("3", msg.getData().get("foo")); + assertEquals("5", msg.getData().get("bar")); + + assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg)); + + msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA); + assertEquals("3", msg.getData().get("foo")); + assertEquals("5", msg.getData().get("bar")); + } + } + + + public class testVisitorInfoMessage implements RunnableTest { + + @Override + public void run() { + VisitorInfoMessage msg = new VisitorInfoMessage(); + msg.getFinishedBuckets().add(new BucketId(16, 1)); + msg.getFinishedBuckets().add(new BucketId(16, 2)); + msg.getFinishedBuckets().add(new BucketId(16, 4)); + msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6"); + assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang); + assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1))); + assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2))); + assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4))); + assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage()); + } + } + } + + public class testSearchResultMessage implements RunnableTest { + + @Override + public void run() throws Exception { + FileInputStream stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-1.dat")); + byte[] data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + Routable routable = decode(data); + assertTrue(routable instanceof SearchResultMessage); + + SearchResultMessage msg = (SearchResultMessage)routable; + assertEquals(0, msg.getResult().getHitCount()); + + stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-2.dat")); + data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + routable = decode(data); + assertTrue(routable instanceof SearchResultMessage); + + msg = (SearchResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + + stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-3.dat")); + data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + routable = decode(data); + assertTrue(routable instanceof SearchResultMessage); + + msg = (SearchResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + + stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-4.dat")); + data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + routable = decode(data); + assertTrue(routable instanceof SearchResultMessage); + + msg = (SearchResultMessage)routable; + assertEquals(3, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; + for (int i = 0; i < b.length; i++) { + assertEquals(e[i], b[i]); + } + h = msg.getResult().getHit(1); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; + for (int i = 0; i < b.length; i++) { + assertEquals(d[i], b[i]); + } + h = msg.getResult().getHit(2); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(90.0, h.getRank(), 1E-6); + assertEquals("doc18", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; + for (int i = 0; i < b.length; i++) { + assertEquals(c[i], b[i]); + } + } + } + + public class testPutDocumentMessage implements RunnableTest { + + @Override + public void run() { + PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"))); + msg.setTimestamp(666); + assertEquals(BASE_MESSAGE_LENGTH + 41, serialize("PutDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang); + assertEquals("testdoc", msg.getDocumentPut().getDocument().getDataType().getName()); + assertEquals("doc:scheme:", msg.getDocumentPut().getDocument().getId().toString()); + assertEquals(666, msg.getTimestamp()); + } + } + } + + public class testPutDocumentReply implements RunnableTest { + + @Override + public void run() { + WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); + reply.setHighestModificationTimestamp(30); + + assertEquals(13, serialize("PutDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang); + assertNotNull(obj); + assertEquals(30, obj.getHighestModificationTimestamp()); + } + } + } + + public class testUpdateDocumentMessage implements RunnableTest { + + @Override + public void run() { + DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc"); + DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:")); + update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); + UpdateDocumentMessage msg = new UpdateDocumentMessage(update); + msg.setNewTimestamp(777); + msg.setOldTimestamp(666); + + assertEquals(BASE_MESSAGE_LENGTH + 89, serialize("UpdateDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (UpdateDocumentMessage)deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang); + assertEquals(update, msg.getDocumentUpdate()); + assertEquals(777, msg.getNewTimestamp()); + assertEquals(666, msg.getOldTimestamp()); + } + } + } + + public class testUpdateDocumentReply implements RunnableTest { + + @Override + public void run() { + UpdateDocumentReply reply = new UpdateDocumentReply(); + reply.setHighestModificationTimestamp(30); + reply.setWasFound(false); + + assertEquals(14, serialize("UpdateDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang); + assertNotNull(obj); + assertEquals(30, reply.getHighestModificationTimestamp()); + assertEquals(false, obj.wasFound()); + } + } + } + + public class testVisitorInfoReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO); + } + } + + public class testWrongDistributionReply implements RunnableTest { + + @Override + public void run() { + WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2"); + assertEquals(32, serialize("WrongDistributionReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang); + assertEquals("distributor:3 storage:2", reply.getSystemState()); + } + } + } + + public class testRemoveDocumentReply implements RunnableTest { + + @Override + public void run() { + RemoveDocumentReply reply = new RemoveDocumentReply(); + reply.setHighestModificationTimestamp(30); + reply.setWasFound(false); + + assertEquals(14, serialize("RemoveDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang); + assertNotNull(obj); + assertEquals(30, obj.getHighestModificationTimestamp()); + assertEquals(false, obj.wasFound()); + } + } + } + + public class testRemoveLocationReply implements RunnableTest { + + @Override + public void run() { + testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION); + } + } + + public class testSearchResultReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT); + } + } + + public class testStatBucketReply implements RunnableTest { + + @Override + public void run() { + StatBucketReply msg = new StatBucketReply(); + msg.setResults("These are the votes of the Norwegian jury"); + + assertEquals(50, serialize("StatBucketReply", msg)); + + for (Language lang : LANGUAGES) { + msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang); + assertEquals("These are the votes of the Norwegian jury", msg.getResults()); + } + } + } + + public class testBatchDocumentUpdateMessage implements RunnableTest { + + @Override + public void run() { + DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc"); + BatchDocumentUpdateMessage msg = new BatchDocumentUpdateMessage(1234); + + { + DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("userdoc:footype:1234:foo")); + update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); + msg.addUpdate(update); + } + { + DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc(32,17):footype:1234:123456789:foo")); + update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); + msg.addUpdate(update); + } + + try { + DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("orderdoc:footype:5678:foo")); + update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); + msg.addUpdate(update); + fail(); + } catch (Exception e) { + + } + + try { + DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("groupdoc:footype:hable:foo")); + update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); + msg.addUpdate(update); + fail(); + } catch (Exception e) { + + } + + assertEquals(2, msg.getUpdates().size()); + + assertEquals(BASE_MESSAGE_LENGTH + 202, serialize("BatchDocumentUpdateMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (BatchDocumentUpdateMessage)deserialize("BatchDocumentUpdateMessage", DocumentProtocol.MESSAGE_BATCHDOCUMENTUPDATE, lang); + assertEquals(2, msg.getUpdates().size()); + } + } + } + + public class testBatchDocumentUpdateReply implements RunnableTest { + + @Override + public void run() { + BatchDocumentUpdateReply reply = new BatchDocumentUpdateReply(); + reply.setHighestModificationTimestamp(30); + reply.getDocumentsNotFound().add(false); + reply.getDocumentsNotFound().add(true); + reply.getDocumentsNotFound().add(true); + + assertEquals(20, serialize("BatchDocumentUpdateReply", reply)); + + for (Language lang : LANGUAGES) { + BatchDocumentUpdateReply obj = (BatchDocumentUpdateReply)deserialize("BatchDocumentUpdateReply", DocumentProtocol.REPLY_BATCHDOCUMENTUPDATE, lang); + assertNotNull(obj); + assertEquals(30, obj.getHighestModificationTimestamp()); + assertEquals(3, obj.getDocumentsNotFound().size()); + assertFalse(obj.getDocumentsNotFound().get(0)); + assertTrue(obj.getDocumentsNotFound().get(1)); + assertTrue(obj.getDocumentsNotFound().get(2)); + } + } + } + + + public class testQueryResultReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT); + } + } + + public class testQueryResultMessage implements RunnableTest { + + @Override + public void run() throws Exception { + FileInputStream stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-1.dat")); + byte[] data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + Routable routable = decode(data); + assertTrue(routable instanceof QueryResultMessage); + + QueryResultMessage msg = (QueryResultMessage)routable; + assertEquals(0, msg.getResult().getHitCount()); + + stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-2.dat")); + data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + routable = decode(data); + assertTrue(routable instanceof QueryResultMessage); + + msg = (QueryResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + + stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-3.dat")); + data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + routable = decode(data); + assertTrue(routable instanceof QueryResultMessage); + + msg = (QueryResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + + stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-4.dat")); + data = new byte[stream.available()]; + assertEquals(data.length, stream.read(data)); + + routable = decode(data); + assertTrue(routable instanceof QueryResultMessage); + + msg = (QueryResultMessage)routable; + assertEquals(3, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; + for (int i = 0; i < b.length; i++) { + assertEquals(e[i], b[i]); + } + h = msg.getResult().getHit(1); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; + for (int i = 0; i < b.length; i++) { + assertEquals(d[i], b[i]); + } + h = msg.getResult().getHit(2); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(90.0, h.getRank(), 1E-6); + assertEquals("doc18", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; + for (int i = 0; i < b.length; i++) { + assertEquals(c[i], b[i]); + } + } + } + + public class testGetBucketListReply implements RunnableTest { + + public void run() { + GetBucketListReply reply = new GetBucketListReply(); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); + + assertEquals(56, serialize("GetBucketListReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang); + assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); + assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); + assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); + } + } + } + + public class testGetBucketStateReply implements RunnableTest { + + public void run() { + GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo")); + GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar")); + + GetBucketStateReply reply = new GetBucketStateReply(); + List<DocumentState> state = new ArrayList<>(2); + state.add(new DocumentState(foo, 777, false)); + state.add(new DocumentState(bar, 888, true)); + reply.setBucketState(state); + assertEquals(53, serialize("GetBucketStateReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang); + assertEquals(777, reply.getBucketState().get(0).getTimestamp()); + assertEquals(foo, reply.getBucketState().get(0).getGid()); + assertEquals(false, reply.getBucketState().get(0).isRemoveEntry()); + assertEquals(888, reply.getBucketState().get(1).getTimestamp()); + assertEquals(bar, reply.getBucketState().get(1).getGid()); + assertEquals(true, reply.getBucketState().get(1).isRemoveEntry()); + } + } + } + + public class testGetDocumentReply implements RunnableTest { + + public void run() { + GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")); + assertEquals(43, serialize("GetDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang); + assertEquals("testdoc", reply.getDocument().getDataType().getName()); + assertEquals("doc:scheme:", reply.getDocument().getId().toString()); + } + } + } + + public class testMapVisitorReply implements RunnableTest { + + public void run() { + testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR); + } + } + + protected void testDocumentReply(String filename, int type) { + DocumentReply reply = new DocumentReply(type); + assertEquals(5, serialize(filename, reply)); + + for (Language lang : LANGUAGES) { + reply = (DocumentReply)deserialize(filename, type, lang); + assertNotNull(reply); + } + } + + protected void testVisitorReply(String filename, int type) { + VisitorReply reply = new VisitorReply(type); + assertEquals(5, serialize(filename, reply)); + + for (Language lang : LANGUAGES) { + reply = (VisitorReply)deserialize(filename, type, lang); + assertNotNull(reply); + } + } + +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java index 68448adb8fc..aa97355e0c4 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java @@ -1,121 +1,121 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.yahoo.component.Version;
-import com.yahoo.document.BucketId;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.select.OrderingSpecification;
-import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage;
-import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage;
-import com.yahoo.text.Utf8;
-
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class Messages51TestCase extends Messages50TestCase {
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Setup
- //
- ///////////////////////////////////////////////////////////////////////////////
-
- @Override
- protected void registerTests(Map<Integer, RunnableTest> out) {
- super.registerTests(out);
-
- // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
- // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
- out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage());
- out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage());
- out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply());
- }
-
- @Override
- protected Version version() {
- return new Version(5, 1);
- }
-
- @Override
- protected boolean shouldTestCoverage() {
- return true;
- }
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Tests
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- private static int BASE_MESSAGE_LENGTH = 5;
-
- public class testCreateVisitorMessage implements RunnableTest {
-
- @Override
- public void run() {
- CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london");
- msg.setDocumentSelection("true and false or true");
- msg.getParameters().put("myvar", Utf8.toBytes("somevalue"));
- msg.getParameters().put("anothervar", Utf8.toBytes("34"));
- msg.getBuckets().add(new BucketId(16, 1234));
- msg.setVisitRemoves(true);
- msg.setFieldSet("foo bar");
- msg.setVisitorOrdering(OrderingSpecification.DESCENDING);
- msg.setMaxBucketsPerVisitor(2);
- assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang);
- assertEquals("SomeLibrary", msg.getLibraryName());
- assertEquals("myvisitor", msg.getInstanceId());
- assertEquals("newyork", msg.getControlDestination());
- assertEquals("london", msg.getDataDestination());
- assertEquals("true and false or true", msg.getDocumentSelection());
- assertEquals(8, msg.getMaxPendingReplyCount());
- assertEquals(true, msg.getVisitRemoves());
- assertEquals("foo bar", msg.getFieldSet());
- assertEquals(false, msg.getVisitInconsistentBuckets());
- assertEquals(1, msg.getBuckets().size());
- assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next());
- assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar")));
- assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar")));
- assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering());
- assertEquals(2, msg.getMaxBucketsPerVisitor());
- }
- }
- }
-
- public class testGetDocumentMessage implements RunnableTest {
-
- @Override
- public void run() {
- GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar");
- assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang);
- assertEquals("doc:scheme:", msg.getDocumentId().toString());
- assertEquals("foo bar", msg.getFieldSet());
- }
- }
- }
-
- public class testDocumentIgnoredReply implements RunnableTest {
-
- @Override
- public void run() {
- DocumentIgnoredReply reply = new DocumentIgnoredReply();
- assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply));
-
- for (Language lang : LANGUAGES) {
- reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang);
- }
- }
- }
-}
+package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.component.Version; +import com.yahoo.document.BucketId; +import com.yahoo.document.DocumentId; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.text.Utf8; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Messages51TestCase extends Messages50TestCase { + + //////////////////////////////////////////////////////////////////////////////// + // + // Setup + // + /////////////////////////////////////////////////////////////////////////////// + + @Override + protected void registerTests(Map<Integer, RunnableTest> out) { + super.registerTests(out); + + // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support + // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage()); + out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply()); + } + + @Override + protected Version version() { + return new Version(5, 1); + } + + @Override + protected boolean shouldTestCoverage() { + return true; + } + + //////////////////////////////////////////////////////////////////////////////// + // + // Tests + // + //////////////////////////////////////////////////////////////////////////////// + + private static int BASE_MESSAGE_LENGTH = 5; + + public class testCreateVisitorMessage implements RunnableTest { + + @Override + public void run() { + CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london"); + msg.setDocumentSelection("true and false or true"); + msg.getParameters().put("myvar", Utf8.toBytes("somevalue")); + msg.getParameters().put("anothervar", Utf8.toBytes("34")); + msg.getBuckets().add(new BucketId(16, 1234)); + msg.setVisitRemoves(true); + msg.setFieldSet("foo bar"); + msg.setVisitorOrdering(OrderingSpecification.DESCENDING); + msg.setMaxBucketsPerVisitor(2); + assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang); + assertEquals("SomeLibrary", msg.getLibraryName()); + assertEquals("myvisitor", msg.getInstanceId()); + assertEquals("newyork", msg.getControlDestination()); + assertEquals("london", msg.getDataDestination()); + assertEquals("true and false or true", msg.getDocumentSelection()); + assertEquals(8, msg.getMaxPendingReplyCount()); + assertEquals(true, msg.getVisitRemoves()); + assertEquals("foo bar", msg.getFieldSet()); + assertEquals(false, msg.getVisitInconsistentBuckets()); + assertEquals(1, msg.getBuckets().size()); + assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next()); + assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar"))); + assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar"))); + assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering()); + assertEquals(2, msg.getMaxBucketsPerVisitor()); + } + } + } + + public class testGetDocumentMessage implements RunnableTest { + + @Override + public void run() { + GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar"); + assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang); + assertEquals("doc:scheme:", msg.getDocumentId().toString()); + assertEquals("foo bar", msg.getFieldSet()); + } + } + } + + public class testDocumentIgnoredReply implements RunnableTest { + + @Override + public void run() { + DocumentIgnoredReply reply = new DocumentIgnoredReply(); + assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang); + } + } + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java index 8198044a25e..e98bdd35254 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java @@ -1,108 +1,108 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.google.common.annotations.Beta;
-import com.yahoo.component.Version;
-import com.yahoo.document.*;
-import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate;
-import com.yahoo.documentapi.messagebus.protocol.*;
-
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author Vegard Sjonfjell
- */
-
-@Beta
-public class Messages52TestCase extends Messages51TestCase {
-
- @Override
- protected Version version() {
- return new Version(5, 115, 0);
- }
-
- @Override
- protected boolean shouldTestCoverage() {
- return true;
- }
-
- @Override
- protected void registerTests(Map<Integer, RunnableTest> out) {
- super.registerTests(out);
-
- // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support
- // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now.
-
- out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage());
- out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage());
- }
-
- private static int BASE_MESSAGE_LENGTH = 5;
- private static String CONDITION_STRING = "There's just one condition";
-
- public class testPutDocumentMessage implements RunnableTest {
- @Override
- public void run() {
- PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")));
-
- msg.setTimestamp(666);
- msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
-
- assertEquals(BASE_MESSAGE_LENGTH + 41 + serializedLength(msg.getCondition().getSelection()), serialize("PutDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- final PutDocumentMessage deserializedMsg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang);
- assertEquals(msg.getDocumentPut().getDocument().getDataType().getName(), deserializedMsg.getDocumentPut().getDocument().getDataType().getName());
- assertEquals(msg.getDocumentPut().getDocument().getId().toString(), deserializedMsg.getDocumentPut().getDocument().getId().toString());
- assertEquals(msg.getTimestamp(), deserializedMsg.getTimestamp());
- assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());
- }
- }
- }
-
- public class testUpdateDocumentMessage implements RunnableTest {
- @Override
- public void run() {
- DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc");
- DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:"));
- update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0"));
-
- final UpdateDocumentMessage msg = new UpdateDocumentMessage(update);
- msg.setNewTimestamp(777);
- msg.setOldTimestamp(666);
- msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
-
- assertEquals(BASE_MESSAGE_LENGTH + 89 + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- final UpdateDocumentMessage deserializedMsg = (UpdateDocumentMessage) deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang);
- assertEquals(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate());
- assertEquals(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp());
- assertEquals(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp());
- assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection());
- }
- }
- }
-
- public class testRemoveDocumentMessage implements RunnableTest {
- @Override
- public void run() {
- final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
- msg.setCondition(new TestAndSetCondition(CONDITION_STRING));
-
- assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg));
-
- for (Language lang : LANGUAGES) {
- final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang);
- assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString());
- }
- }
- }
-
- static int serializedLength(String str) {
- return 4 + str.length();
- }
-}
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi.messagebus.protocol.test; + +import com.google.common.annotations.Beta; +import com.yahoo.component.Version; +import com.yahoo.document.*; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; +import com.yahoo.documentapi.messagebus.protocol.*; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author Vegard Sjonfjell + */ + +@Beta +public class Messages52TestCase extends Messages51TestCase { + + @Override + protected Version version() { + return new Version(5, 115, 0); + } + + @Override + protected boolean shouldTestCoverage() { + return true; + } + + @Override + protected void registerTests(Map<Integer, RunnableTest> out) { + super.registerTests(out); + + // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support + // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + + out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage()); + } + + private static int BASE_MESSAGE_LENGTH = 5; + private static String CONDITION_STRING = "There's just one condition"; + + public class testPutDocumentMessage implements RunnableTest { + @Override + public void run() { + PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"))); + + msg.setTimestamp(666); + msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); + + assertEquals(BASE_MESSAGE_LENGTH + 41 + serializedLength(msg.getCondition().getSelection()), serialize("PutDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + final PutDocumentMessage deserializedMsg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang); + assertEquals(msg.getDocumentPut().getDocument().getDataType().getName(), deserializedMsg.getDocumentPut().getDocument().getDataType().getName()); + assertEquals(msg.getDocumentPut().getDocument().getId().toString(), deserializedMsg.getDocumentPut().getDocument().getId().toString()); + assertEquals(msg.getTimestamp(), deserializedMsg.getTimestamp()); + assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); + } + } + } + + public class testUpdateDocumentMessage implements RunnableTest { + @Override + public void run() { + DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc"); + DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:")); + update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); + + final UpdateDocumentMessage msg = new UpdateDocumentMessage(update); + msg.setNewTimestamp(777); + msg.setOldTimestamp(666); + msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); + + assertEquals(BASE_MESSAGE_LENGTH + 89 + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + final UpdateDocumentMessage deserializedMsg = (UpdateDocumentMessage) deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang); + assertEquals(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate()); + assertEquals(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp()); + assertEquals(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp()); + assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); + } + } + } + + public class testRemoveDocumentMessage implements RunnableTest { + @Override + public void run() { + final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); + msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); + + assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang); + assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString()); + } + } + } + + static int serializedLength(String str) { + return 4 + str.length(); + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java index f69454a99f4..7b7ec353284 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java @@ -1,161 +1,161 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.yahoo.component.Version;
-import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.document.DocumentTypeManagerConfigurer;
-import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.messagebus.Routable;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.*;
-
-import static org.junit.Assert.*;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public abstract class MessagesTestBase {
-
- protected enum Language {
- JAVA,
- CPP
- }
- protected static final Set<Language> LANGUAGES = EnumSet.allOf(Language.class);
-
- protected final DocumentTypeManager docMan = new DocumentTypeManager();
- protected final LoadTypeSet loadTypes = new LoadTypeSet();
- protected final DocumentProtocol protocol = new DocumentProtocol(docMan, null, loadTypes);
-
- public MessagesTestBase() {
- DocumentTypeManagerConfigurer.configure(docMan, "file:./test/cfg/testdoc.cfg");
- loadTypes.addLoadType(34, "foo", DocumentProtocol.Priority.NORMAL_2);
- }
-
- @Test
- public void requireThatTestsPass() throws Exception {
- Map<Integer, RunnableTest> tests = new TreeMap<>();
- registerTests(tests);
- for (Map.Entry<Integer, RunnableTest> entry : tests.entrySet()) {
- entry.getValue().run();
- }
- if (shouldTestCoverage()) {
- assertCoverage(protocol.getRoutableTypes(version()), new ArrayList<>(tests.keySet()));
- }
- }
-
- /**
- * Returns the version to use for serialization.
- *
- * @return The version.
- */
- protected abstract Version version();
-
- /**
- * Registers the tests to run.
- */
- protected abstract void registerTests(Map<Integer, RunnableTest> out);
-
- /**
- * Returns whether or not to test message test coverage.
- */
- protected abstract boolean shouldTestCoverage();
-
- /**
- * Encodes the given routable using the current version of the test case.
- *
- * @param routable The routable to encode.
- * @return The encoded data.
- */
- public byte[] encode(Routable routable) {
- return protocol.encode(version(), routable);
- }
-
- /**
- * Decodes the given byte array using the current version of the test case.
- *
- * @param data The data to decode.
- * @return The decoded routable.
- */
- public Routable decode(byte[] data) {
- return protocol.decode(version(), data);
- }
-
- public String getPath(String filename) {
- return TestFileUtil.getPath(filename);
- }
-
- /**
- * Writes the content of the given routable to the given file.
- *
- * @param filename The name of the file to write to.
- * @param routable The routable to serialize.
- * @return The size of the written file.
- */
- public int serialize(String filename, Routable routable) {
- Version version = version();
- String path = getPath(version + "-java-" + filename + ".dat");
- System.out.println("Serializing to '" + path + "'..");
- byte[] data = protocol.encode(version, routable);
- assertNotNull(data);
- assertTrue(data.length > 0);
- try {
- TestFileUtil.writeToFile(path, data);
- } catch (IOException e) {
- throw new AssertionError(e);
- }
- assertEquals(routable.getType(), protocol.decode(version, data).getType());
- return data.length;
- }
-
- /**
- * Reads the content of the given file and creates a corresponding routable.
- *
- * @param filename The name of the file to read from.
- * @param classId The type that the routable must decode as.
- * @param lang The language constant that dictates what file format to read from.
- * @return The decoded routable.
- */
- public Routable deserialize(String filename, int classId, Language lang) {
- Version version = version();
- String path = getPath(version + "-" + (lang == Language.JAVA ? "java" : "cpp") + "-" + filename + ".dat");
- System.out.println("Deserializing from '" + path + "'..");
- byte[] data;
- try {
- data = TestFileUtil.readFile(path);
- } catch (IOException e) {
- throw new AssertionError(e);
- }
- Routable ret = protocol.decode(version, data);
- assertNotNull(ret);
- assertEquals(classId, ret.getType());
- return ret;
- }
-
- private static void assertCoverage(List<Integer> registered, List<Integer> tested) {
- boolean ok = true;
- List<Integer> lst = new ArrayList<>(tested);
- for (Integer type : registered) {
- if (!lst.contains(type)) {
- System.err.println("Routable type " + type + " is registered in DocumentProtocol but not tested.");
- ok = false;
- } else {
- lst.remove(type);
- }
- }
- if (!lst.isEmpty()) {
- for (Integer type : lst) {
- System.err.println("Routable type " + type + " is tested but not registered in DocumentProtocol.");
- }
- ok = false;
- }
- assertTrue(ok);
- }
-
- protected static interface RunnableTest {
-
- public void run() throws Exception;
- }
-}
+package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.component.Version; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.messagebus.Routable; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class MessagesTestBase { + + protected enum Language { + JAVA, + CPP + } + protected static final Set<Language> LANGUAGES = EnumSet.allOf(Language.class); + + protected final DocumentTypeManager docMan = new DocumentTypeManager(); + protected final LoadTypeSet loadTypes = new LoadTypeSet(); + protected final DocumentProtocol protocol = new DocumentProtocol(docMan, null, loadTypes); + + public MessagesTestBase() { + DocumentTypeManagerConfigurer.configure(docMan, "file:./test/cfg/testdoc.cfg"); + loadTypes.addLoadType(34, "foo", DocumentProtocol.Priority.NORMAL_2); + } + + @Test + public void requireThatTestsPass() throws Exception { + Map<Integer, RunnableTest> tests = new TreeMap<>(); + registerTests(tests); + for (Map.Entry<Integer, RunnableTest> entry : tests.entrySet()) { + entry.getValue().run(); + } + if (shouldTestCoverage()) { + assertCoverage(protocol.getRoutableTypes(version()), new ArrayList<>(tests.keySet())); + } + } + + /** + * Returns the version to use for serialization. + * + * @return The version. + */ + protected abstract Version version(); + + /** + * Registers the tests to run. + */ + protected abstract void registerTests(Map<Integer, RunnableTest> out); + + /** + * Returns whether or not to test message test coverage. + */ + protected abstract boolean shouldTestCoverage(); + + /** + * Encodes the given routable using the current version of the test case. + * + * @param routable The routable to encode. + * @return The encoded data. + */ + public byte[] encode(Routable routable) { + return protocol.encode(version(), routable); + } + + /** + * Decodes the given byte array using the current version of the test case. + * + * @param data The data to decode. + * @return The decoded routable. + */ + public Routable decode(byte[] data) { + return protocol.decode(version(), data); + } + + public String getPath(String filename) { + return TestFileUtil.getPath(filename); + } + + /** + * Writes the content of the given routable to the given file. + * + * @param filename The name of the file to write to. + * @param routable The routable to serialize. + * @return The size of the written file. + */ + public int serialize(String filename, Routable routable) { + Version version = version(); + String path = getPath(version + "-java-" + filename + ".dat"); + System.out.println("Serializing to '" + path + "'.."); + byte[] data = protocol.encode(version, routable); + assertNotNull(data); + assertTrue(data.length > 0); + try { + TestFileUtil.writeToFile(path, data); + } catch (IOException e) { + throw new AssertionError(e); + } + assertEquals(routable.getType(), protocol.decode(version, data).getType()); + return data.length; + } + + /** + * Reads the content of the given file and creates a corresponding routable. + * + * @param filename The name of the file to read from. + * @param classId The type that the routable must decode as. + * @param lang The language constant that dictates what file format to read from. + * @return The decoded routable. + */ + public Routable deserialize(String filename, int classId, Language lang) { + Version version = version(); + String path = getPath(version + "-" + (lang == Language.JAVA ? "java" : "cpp") + "-" + filename + ".dat"); + System.out.println("Deserializing from '" + path + "'.."); + byte[] data; + try { + data = TestFileUtil.readFile(path); + } catch (IOException e) { + throw new AssertionError(e); + } + Routable ret = protocol.decode(version, data); + assertNotNull(ret); + assertEquals(classId, ret.getType()); + return ret; + } + + private static void assertCoverage(List<Integer> registered, List<Integer> tested) { + boolean ok = true; + List<Integer> lst = new ArrayList<>(tested); + for (Integer type : registered) { + if (!lst.contains(type)) { + System.err.println("Routable type " + type + " is registered in DocumentProtocol but not tested."); + ok = false; + } else { + lst.remove(type); + } + } + if (!lst.isEmpty()) { + for (Integer type : lst) { + System.err.println("Routable type " + type + " is tested but not registered in DocumentProtocol."); + } + ok = false; + } + assertTrue(ok); + } + + protected static interface RunnableTest { + + public void run() throws Exception; + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java index 3bbba1c0984..a2f5293bb98 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyFactoryTestCase.java @@ -1,124 +1,124 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy;
-import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
-import com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory;
-import com.yahoo.jrt.ListenFailedException;
-import com.yahoo.jrt.slobrok.server.Slobrok;
-import com.yahoo.messagebus.*;
-import com.yahoo.messagebus.metrics.MetricSet;
-import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
-import com.yahoo.messagebus.network.rpc.test.TestServer;
-import com.yahoo.messagebus.routing.Route;
-import com.yahoo.messagebus.routing.RoutingContext;
-import com.yahoo.messagebus.test.Receptor;
-import com.yahoo.text.Utf8Array;
-import junit.framework.TestCase;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class PolicyFactoryTestCase extends TestCase {
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Setup
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- private Slobrok slobrok;
- private TestServer srv;
- private SourceSession src;
-
- @Override
- public void setUp() throws ListenFailedException {
- slobrok = new Slobrok();
- srv = new TestServer(new MessageBusParams().addProtocol(new DocumentProtocol(new DocumentTypeManager())),
- new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
- src = srv.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor()));
- }
-
- @Override
- public void tearDown() {
- slobrok.stop();
- src.destroy();
- srv.destroy();
- }
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Tests
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- public void testFactory() {
- Route route = Route.parse("[MyPolicy]");
- assertTrue(src.send(createMessage(), route).isAccepted());
- Reply reply = ((Receptor)src.getReplyHandler()).getReply(60);
- assertNotNull(reply);
- System.out.println(reply.getTrace());
- assertEquals(1, reply.getNumErrors());
- assertEquals(ErrorCode.UNKNOWN_POLICY, reply.getError(0).getCode());
-
- Protocol obj = srv.mb.getProtocol(DocumentProtocol.NAME);
- assertTrue(obj instanceof DocumentProtocol);
- DocumentProtocol protocol = (DocumentProtocol)obj;
- protocol.putRoutingPolicyFactory("MyPolicy", new MyFactory());
-
- assertTrue(src.send(createMessage(), route).isAccepted());
- assertNotNull(reply = ((Receptor)src.getReplyHandler()).getReply(60));
- System.out.println(reply.getTrace());
- assertEquals(1, reply.getNumErrors());
- assertEquals(DocumentProtocol.ERROR_POLICY_FAILURE, reply.getError(0).getCode());
- }
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Utilities
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- private static Message createMessage() {
- Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
- msg.getTrace().setLevel(9);
- return msg;
- }
-
- private static class MyFactory implements RoutingPolicyFactory {
-
- public DocumentProtocolRoutingPolicy createPolicy(String param) {
- return new MyPolicy(param);
- }
-
- public void destroy() {
- }
- }
-
- private static class MyPolicy implements DocumentProtocolRoutingPolicy {
-
- private final String param;
-
- private MyPolicy(String param) {
- this.param = param;
- }
-
- public void select(RoutingContext ctx) {
- ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, param);
- }
-
- public void merge(RoutingContext ctx) {
- throw new AssertionError("Routing passed terminated select.");
- }
-
- public void destroy() {
- }
-
- public MetricSet getMetrics() {
- return null;
- }
- }
-}
+package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.slobrok.server.Slobrok; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.metrics.MetricSet; +import com.yahoo.messagebus.network.rpc.RPCNetworkParams; +import com.yahoo.messagebus.network.rpc.test.TestServer; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingContext; +import com.yahoo.messagebus.test.Receptor; +import com.yahoo.text.Utf8Array; +import junit.framework.TestCase; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class PolicyFactoryTestCase extends TestCase { + + //////////////////////////////////////////////////////////////////////////////// + // + // Setup + // + //////////////////////////////////////////////////////////////////////////////// + + private Slobrok slobrok; + private TestServer srv; + private SourceSession src; + + @Override + public void setUp() throws ListenFailedException { + slobrok = new Slobrok(); + srv = new TestServer(new MessageBusParams().addProtocol(new DocumentProtocol(new DocumentTypeManager())), + new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); + src = srv.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor())); + } + + @Override + public void tearDown() { + slobrok.stop(); + src.destroy(); + srv.destroy(); + } + + //////////////////////////////////////////////////////////////////////////////// + // + // Tests + // + //////////////////////////////////////////////////////////////////////////////// + + public void testFactory() { + Route route = Route.parse("[MyPolicy]"); + assertTrue(src.send(createMessage(), route).isAccepted()); + Reply reply = ((Receptor)src.getReplyHandler()).getReply(60); + assertNotNull(reply); + System.out.println(reply.getTrace()); + assertEquals(1, reply.getNumErrors()); + assertEquals(ErrorCode.UNKNOWN_POLICY, reply.getError(0).getCode()); + + Protocol obj = srv.mb.getProtocol(DocumentProtocol.NAME); + assertTrue(obj instanceof DocumentProtocol); + DocumentProtocol protocol = (DocumentProtocol)obj; + protocol.putRoutingPolicyFactory("MyPolicy", new MyFactory()); + + assertTrue(src.send(createMessage(), route).isAccepted()); + assertNotNull(reply = ((Receptor)src.getReplyHandler()).getReply(60)); + System.out.println(reply.getTrace()); + assertEquals(1, reply.getNumErrors()); + assertEquals(DocumentProtocol.ERROR_POLICY_FAILURE, reply.getError(0).getCode()); + } + + //////////////////////////////////////////////////////////////////////////////// + // + // Utilities + // + //////////////////////////////////////////////////////////////////////////////// + + private static Message createMessage() { + Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); + msg.getTrace().setLevel(9); + return msg; + } + + private static class MyFactory implements RoutingPolicyFactory { + + public DocumentProtocolRoutingPolicy createPolicy(String param) { + return new MyPolicy(param); + } + + public void destroy() { + } + } + + private static class MyPolicy implements DocumentProtocolRoutingPolicy { + + private final String param; + + private MyPolicy(String param) { + this.param = param; + } + + public void select(RoutingContext ctx) { + ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, param); + } + + public void merge(RoutingContext ctx) { + throw new AssertionError("Routing passed terminated select."); + } + + public void destroy() { + } + + public MetricSet getMetrics() { + return null; + } + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java index ef1cdd2818e..82573600d4c 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java @@ -1,902 +1,902 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.yahoo.document.*;
-import com.yahoo.documentapi.messagebus.protocol.*;
-import com.yahoo.jrt.ListenFailedException;
-import com.yahoo.jrt.slobrok.api.IMirror;
-import com.yahoo.jrt.slobrok.api.Mirror;
-import com.yahoo.jrt.slobrok.server.Slobrok;
-import com.yahoo.messagebus.*;
-import com.yahoo.messagebus.Error;
-import com.yahoo.messagebus.network.rpc.test.TestServer;
-import com.yahoo.messagebus.routing.*;
-import com.yahoo.messagebus.test.Receptor;
-import com.yahoo.vdslib.DocumentList;
-import com.yahoo.vdslib.Entry;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-@SuppressWarnings("deprecation")
-public class PolicyTestCase {
-
- private static final int TIMEOUT = 300;
- private static final TimeUnit TIMEOUT_UNIT = TimeUnit.SECONDS;
- private static final long TIMEOUT_MILLIS = TIMEOUT_UNIT.toMillis(TIMEOUT);
- private final DocumentTypeManager manager = new DocumentTypeManager();
-
- @Before
- public void setUp() {
- DocumentTypeManagerConfigurer.configure(manager, "file:./test/cfg/testdoc.cfg");
- }
-
- @Test
- public void testProtocol() {
- DocumentProtocol protocol = new DocumentProtocol(manager);
-
- RoutingPolicy policy = protocol.createPolicy("AND", null);
- assertTrue(policy instanceof ANDPolicy);
-
- policy = new DocumentProtocol(manager).createPolicy("DocumentRouteSelector", "raw:route[0]\n");
- assertTrue(policy instanceof DocumentRouteSelectorPolicy);
-
- policy = new DocumentProtocol(manager).createPolicy("Extern", "foo;bar/baz");
- assertTrue(policy instanceof ExternPolicy);
-
- policy = new DocumentProtocol(manager).createPolicy("LocalService", null);
- assertTrue(policy instanceof LocalServicePolicy);
-
- policy = new DocumentProtocol(manager).createPolicy("RoundRobin", null);
- assertTrue(policy instanceof RoundRobinPolicy);
-
- policy = new DocumentProtocol(manager).createPolicy("SearchRow", null);
- assertTrue(policy instanceof SearchRowPolicy);
-
- policy = new DocumentProtocol(manager).createPolicy("SearchColumn", null);
- assertTrue(policy instanceof SearchColumnPolicy);
-
- policy = new DocumentProtocol(manager).createPolicy("SubsetService", null);
- assertTrue(policy instanceof SubsetServicePolicy);
-
- policy = new DocumentProtocol(manager).createPolicy("LoadBalancer", null);
- assertTrue(policy instanceof LoadBalancerPolicy);
- }
-
- @Test
- public void testAND() {
- PolicyTestFrame frame = new PolicyTestFrame(manager);
- frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:")))));
- frame.setHop(new HopSpec("test", "[AND]")
- .addRecipient("foo")
- .addRecipient("bar"));
- frame.assertSelect(Arrays.asList("foo", "bar"));
-
- frame.setHop(new HopSpec("test", "[AND:baz]")
- .addRecipient("foo")
- .addRecipient("bar"));
- frame.assertSelect(Arrays.asList("baz")); // param precedes recipients
-
- frame.setHop(new HopSpec("test", "[AND:foo]"));
- frame.assertMergeOneReply("foo");
-
- frame.setHop(new HopSpec("test", "[AND:foo bar]"));
- frame.assertMergeTwoReplies("foo", "bar");
- frame.destroy();
- }
-
- @Test
- public void requireThatExternPolicyWithIllegalParamIsAnErrorPolicy() throws ListenFailedException {
- Slobrok slobrok = new Slobrok();
- String spec = "tcp/localhost:" + slobrok.port();
- assertTrue(new DocumentProtocol(manager).createPolicy("Extern", null) instanceof ErrorPolicy);
- assertTrue(new DocumentProtocol(manager).createPolicy("Extern", "") instanceof ErrorPolicy);
- assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec) instanceof ErrorPolicy);
- assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";") instanceof ErrorPolicy);
- assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";bar") instanceof ErrorPolicy);
- }
-
- @Test
- public void requireThatExternPolicyWithUnknownPatternSelectsNone() throws Exception {
- PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");
- setupExternPolicy(frame, new Slobrok(), "foo/bar");
- frame.assertSelect(null);
- }
-
- @Test
- public void requireThatExternPolicySelectsFromExternSlobrok() throws Exception {
- PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");
- Slobrok slobrok = new Slobrok();
- List<TestServer> servers = new ArrayList<>();
- for (int i = 0; i < 10; ++i) {
- TestServer server = new TestServer("docproc/cluster.default/" + i, null, slobrok, null,
- new DocumentProtocol(manager));
- server.net.registerSession("chain.default");
- servers.add(server);
- }
- setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 10);
- Set<String> lst = new HashSet<>();
- for (int i = 0; i < 10; ++i) {
- RoutingNode leaf = frame.select(1).get(0);
- String recipient = leaf.getRoute().toString();
- lst.add(recipient);
-
- leaf.handleReply(new EmptyReply());
- assertNotNull(frame.getReceptor().getReply(TIMEOUT));
- }
- assertEquals(10, lst.size());
- for (TestServer server : servers) {
- server.destroy();
- }
- frame.destroy();
- }
-
- @Test
- public void requireThatExternPolicyMergesOneReplyAsProtocol() throws Exception {
- PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:");
- Slobrok slobrok = new Slobrok();
- TestServer server = new TestServer("docproc/cluster.default/0", null, slobrok, null,
- new DocumentProtocol(manager));
- server.net.registerSession("chain.default");
- setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 1);
- frame.assertMergeOneReply(server.net.getConnectionSpec() + "/chain.default");
- server.destroy();
- frame.destroy();
- }
-
- @Test
- public void testExternSend() throws Exception {
- // Setup local source node.
- Slobrok local = new Slobrok();
- TestServer src = new TestServer("src", null, local, null, new DocumentProtocol(manager));
- SourceSession ss = src.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT));
-
- // Setup remote cluster with routing config.
- Slobrok slobrok = new Slobrok();
- TestServer itr = new TestServer("itr",
- new RoutingTableSpec(DocumentProtocol.NAME)
- .addRoute(new RouteSpec("default").addHop("dst"))
- .addHop(new HopSpec("dst", "dst/session")),
- slobrok, null, new DocumentProtocol(manager));
- IntermediateSession is = itr.mb.createIntermediateSession("session", true, new Receptor(), new Receptor());
- TestServer dst = new TestServer("dst", null, slobrok, null, new DocumentProtocol(manager));
- DestinationSession ds = dst.mb.createDestinationSession("session", true, new Receptor());
-
- // Send message from local node to remote cluster and resolve route there.
- Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
- msg.getTrace().setLevel(9);
- msg.setRoute(Route.parse("[Extern:tcp/localhost:" + slobrok.port() + ";itr/session] default"));
-
- assertTrue(ss.send(msg).isAccepted());
- assertNotNull(msg = ((Receptor)is.getMessageHandler()).getMessage(TIMEOUT));
- is.forward(msg);
- assertNotNull(msg = ((Receptor)ds.getMessageHandler()).getMessage(TIMEOUT));
- ds.acknowledge(msg);
- Reply reply = ((Receptor)is.getReplyHandler()).getReply(TIMEOUT);
- assertNotNull(reply);
- is.forward(reply);
- assertNotNull(reply = ((Receptor)ss.getReplyHandler()).getReply(TIMEOUT));
-
- System.out.println(reply.getTrace().toString());
-
- // Perform necessary cleanup.
- src.destroy();
- itr.destroy();
- dst.destroy();
- slobrok.stop();
- local.stop();
- }
-
- @Test
- public void testExternMultipleSlobroks() throws ListenFailedException {
- Slobrok local = new Slobrok();
- TestServer srcServer = new TestServer("src", null, local, null, new DocumentProtocol(manager));
- SourceSession srcSession =
- srcServer.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT));
-
- Slobrok extern = new Slobrok();
- String spec = "tcp/localhost:" + extern.port();
-
- TestServer dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager));
- Receptor dstHandler = new Receptor();
- DestinationSession dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler);
-
- Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
- msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]"));
- assertTrue(srcSession.send(msg).isAccepted());
- assertNotNull(msg = dstHandler.getMessage(TIMEOUT));
- dstSession.acknowledge(msg);
- Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT);
- assertNotNull(reply);
-
- extern.stop();
- dstSession.destroy();
- dstServer.destroy();
- dstHandler.reset();
- assertNull(dstHandler.getMessage(0));
-
- extern = new Slobrok();
- spec += ",tcp/localhost:" + extern.port();
-
- dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager));
- dstHandler = new Receptor();
- dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler);
-
- msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:"));
- msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]"));
- assertTrue(srcSession.send(msg).isAccepted());
- assertNotNull(msg = dstHandler.getMessage(TIMEOUT));
- dstSession.acknowledge(msg);
- reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT);
- assertNotNull(reply);
-
- extern.stop();
- dstSession.destroy();
- dstServer.destroy();
-
- local.stop();
- srcSession.destroy();
- srcServer.destroy();
- }
-
- @Test
- public void testLocalService() {
- // Test select with proper address.
- PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
- frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:0")))));
- for (int i = 0; i < 10; ++i) {
- frame.getNetwork().registerSession(i + "/chain.default");
- }
- assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10));
- frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService]/chain.default"));
-
- Set<String> lst = new HashSet<>();
- for (int i = 0; i < 10; ++i) {
- RoutingNode leaf = frame.select(1).get(0);
- String recipient = leaf.getRoute().toString();
- lst.add(recipient);
-
- leaf.handleReply(new EmptyReply());
- assertNotNull(frame.getReceptor().getReply(TIMEOUT));
- }
- assertEquals(10, lst.size());
-
- // Test select with broken address.
- lst.clear();
- frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService:broken]/chain.default"));
- for (int i = 0; i < 10; ++i) {
- RoutingNode leaf = frame.select(1).get(0);
- String recipient = leaf.getRoute().toString();
- assertTrue(recipient.equals("docproc/cluster.default/*/chain.default"));
- lst.add(recipient);
-
- leaf.handleReply(new EmptyReply());
- assertNotNull(frame.getReceptor().getReply(TIMEOUT));
- }
- assertEquals(1, lst.size());
-
- // Test merge behavior.
- frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:")))));
- frame.setHop(new HopSpec("test", "[LocalService]"));
- frame.assertMergeOneReply("*");
-
- frame.destroy();
- }
-
- @Test
- public void testLocalServiceCache() {
- PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);
- HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[LocalService]/chain.foo");
- fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));
- fooFrame.setHop(fooHop);
-
- PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);
- HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[LocalService]/chain.bar");
- barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));
- barFrame.setHop(barHop);
-
- fooFrame.getMessageBus().setupRouting(
- new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)
- .addHop(fooHop)
- .addHop(barHop)));
-
- fooFrame.getNetwork().registerSession("0/chain.foo");
- fooFrame.getNetwork().registerSession("0/chain.bar");
- assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));
-
- RoutingNode fooChild = fooFrame.select(1).get(0);
- assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString());
- RoutingNode barChild = barFrame.select(1).get(0);
- assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString());
-
- barChild.handleReply(new EmptyReply());
- fooChild.handleReply(new EmptyReply());
-
- assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));
- assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));
- }
-
- @Test
- public void testSearchRow() {
- PolicyTestFrame frame = new PolicyTestFrame(manager);
- frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:")))));
- frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo"));
- frame.assertMergeOneReply("foo");
- frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar"));
- frame.assertMergeTwoReplies("foo", "bar");
-
- frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo"));
- Map<String, Integer> replies = new HashMap<>();
- replies.put("foo", ErrorCode.SERVICE_OOS);
- frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS));
-
- frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar"));
- replies.put("foo", ErrorCode.SERVICE_OOS);
- replies.put("bar", ErrorCode.NONE);
- frame.assertMergeOk(replies, Arrays.asList("bar"));
-
- replies.put("foo", ErrorCode.SERVICE_OOS);
- replies.put("bar", ErrorCode.SERVICE_OOS);
- frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
-
- frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));
- replies.put("foo", ErrorCode.SERVICE_OOS);
- replies.put("bar", ErrorCode.NONE);
- replies.put("baz", ErrorCode.NONE);
- frame.assertMergeOk(replies, Arrays.asList("bar", "baz"));
-
- replies.put("foo", ErrorCode.SERVICE_OOS);
- replies.put("bar", ErrorCode.SERVICE_OOS);
- replies.put("baz", ErrorCode.NONE);
- frame.assertMergeOk(replies, Arrays.asList("baz"));
-
- replies.put("foo", ErrorCode.SERVICE_OOS);
- replies.put("bar", ErrorCode.SERVICE_OOS);
- replies.put("baz", ErrorCode.SERVICE_OOS);
- frame.assertMergeError(replies,
- Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
-
- frame.setHop(new HopSpec("test", "[SearchRow:2]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));
- replies.put("foo", ErrorCode.SERVICE_OOS);
- replies.put("bar", ErrorCode.NONE);
- replies.put("baz", ErrorCode.NONE);
- frame.assertMergeOk(replies, Arrays.asList("bar", "baz"));
-
- replies.put("foo", ErrorCode.SERVICE_OOS);
- replies.put("bar", ErrorCode.SERVICE_OOS);
- replies.put("baz", ErrorCode.NONE);
- frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
-
- replies.put("foo", ErrorCode.SERVICE_OOS);
- replies.put("bar", ErrorCode.SERVICE_OOS);
- replies.put("baz", ErrorCode.SERVICE_OOS);
- frame.assertMergeError(replies,
- Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS));
-
- frame.destroy();
- }
-
- @Test
- public void testSearchRowMerge() {
- PolicyTestFrame frame = new PolicyTestFrame(manager);
- frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo"));
- tryWasFound(frame, 1, 0x0, false);
- tryWasFound(frame, 1, 0x1, true);
-
- frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar"));
- tryWasFound(frame, 2, 0x0, false);
- tryWasFound(frame, 2, 0x1, true);
- tryWasFound(frame, 2, 0x2, true);
- tryWasFound(frame, 2, 0x3, true);
-
- frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar").addRecipient("baz"));
- tryWasFound(frame, 3, 0x0, false);
- tryWasFound(frame, 3, 0x1, true);
- tryWasFound(frame, 3, 0x2, true);
- tryWasFound(frame, 3, 0x3, true);
- tryWasFound(frame, 3, 0x4, true);
- tryWasFound(frame, 3, 0x5, true);
- tryWasFound(frame, 3, 0x6, true);
- tryWasFound(frame, 3, 0x7, true);
- frame.destroy();
- }
-
- private void tryWasFound(PolicyTestFrame frame, int expectedRecipients,
- int foundMask, boolean expectedFound)
- {
- {
- frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:69")));
- List<RoutingNode> selected = frame.select(expectedRecipients);
- for (int i = 0, len = selected.size(); i < len; ++i) {
- RemoveDocumentReply reply = new RemoveDocumentReply();
- reply.setWasFound(((1 << i) & foundMask) != 0);
- selected.get(i).handleReply(reply);
- }
- Reply reply = frame.getReceptor().getReply(TIMEOUT);
- assertNotNull(reply);
- assertEquals(DocumentProtocol.REPLY_REMOVEDOCUMENT, reply.getType());
- assertEquals(expectedFound, ((RemoveDocumentReply)reply).wasFound());
- }
- {
- DocumentUpdate upd = new DocumentUpdate(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:"));
- frame.setMessage(new UpdateDocumentMessage(upd));
- List<RoutingNode> selected = frame.select(expectedRecipients);
- for (int i = 0, len = selected.size(); i < len; ++i) {
- UpdateDocumentReply reply = new UpdateDocumentReply();
- reply.setWasFound(((1 << i) & foundMask) != 0);
- selected.get(i).handleReply(reply);
- }
- Reply reply = frame.getReceptor().getReply(TIMEOUT);
- assertNotNull(reply);
- assertEquals(DocumentProtocol.REPLY_UPDATEDOCUMENT, reply.getType());
- assertEquals(expectedFound, ((UpdateDocumentReply)reply).wasFound());
- }
- }
-
- @Test
- public void multipleGetRepliesAreMergedToFoundDocument() {
- PolicyTestFrame frame = new PolicyTestFrame(manager);
- frame.setHop(new HopSpec("test", getDocumentRouteSelectorRawConfig())
- .addRecipient("foo").addRecipient("bar"));
- frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:yarn"), "[all]"));
- List<RoutingNode> selected = frame.select(2);
- for (int i = 0, len = selected.size(); i < len; ++i) {
- Document doc = null;
- if (i == 0) {
- doc = new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:yarn"));
- doc.setLastModified(123456L);
- }
- GetDocumentReply reply = new GetDocumentReply(null);
- reply.setDocument(doc);
- selected.get(i).handleReply(reply);
- }
- Reply reply = frame.getReceptor().getReply(TIMEOUT);
- assertNotNull(reply);
- assertEquals(DocumentProtocol.REPLY_GETDOCUMENT, reply.getType());
- assertEquals(123456L, ((GetDocumentReply)reply).getLastModified());
- }
-
- private String getDocumentRouteSelectorRawConfig() {
- return "[DocumentRouteSelector:raw:" +
- "route[2]\n" +
- "route[0].name \"foo\"\n" +
- "route[0].selector \"testdoc\"\n" +
- "route[0].feed \"myfeed\"\n" +
- "route[1].name \"bar\"\n" +
- "route[1].selector \"other\"\n" +
- "route[1].feed \"myfeed\"\n]";
- }
-
- @Test
- public void testSearchColumn() {
- PolicyTestFrame frame = new PolicyTestFrame(manager);
- frame.setHop(new HopSpec("test", "[SearchColumn]")
- .addRecipient("c0").addRecipient("c1")
- .addRecipient("c2").addRecipient("c3"));
-
- // Test hash distribution.
- assertDistribution(frame, "doc:ns:3", "c0");
- assertDistribution(frame, "doc:ns:18", "c1");
- assertDistribution(frame, "doc:ns:0", "c2");
- assertDistribution(frame, "doc:ns:4", "c3");
-
- assertDistribution(frame, "userdoc:ns:49152:0", "c0");
- assertDistribution(frame, "userdoc:ns:49152:1", "c0");
- assertDistribution(frame, "userdoc:ns:16384:2", "c1");
- assertDistribution(frame, "userdoc:ns:16384:3", "c1");
- assertDistribution(frame, "userdoc:ns:5461:4", "c2");
- assertDistribution(frame, "userdoc:ns:5461:5", "c2");
- assertDistribution(frame, "userdoc:ns:0:6", "c3");
- assertDistribution(frame, "userdoc:ns:0:7", "c3");
-
- assertDistribution(frame, "groupdoc:ns:0:0", "c0");
- assertDistribution(frame, "groupdoc:ns:0:1", "c0");
- assertDistribution(frame, "groupdoc:ns:4:2", "c1");
- assertDistribution(frame, "groupdoc:ns:4:3", "c1");
- assertDistribution(frame, "groupdoc:ns:2:4", "c2");
- assertDistribution(frame, "groupdoc:ns:2:5", "c2");
- assertDistribution(frame, "groupdoc:ns:7:6", "c3");
- assertDistribution(frame, "groupdoc:ns:7:7", "c3");
-
- // Test routing based on message type.
- Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:"))));
- frame.setHop(new HopSpec("test", "[SearchColumn]").addRecipient("c0").addRecipient("c1"));
- frame.setMessage(put);
- frame.assertMergeOneReply("c0");
-
- // Test allowed bad parts.
- frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0"));
- frame.setMessage(put);
- Map<String, Integer> replies = new HashMap<>();
- replies.put("c0", ErrorCode.SERVICE_OOS);
- frame.assertMergeOk(replies, null);
-
- replies.put("c0", ErrorCode.SERVICE_OOS);
- frame.assertMergeOk(replies, null);
-
- frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1"));
- frame.setMessage(put);
- replies.put("c0", ErrorCode.SERVICE_OOS);
- frame.assertMergeOk(replies, null);
-
- frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1").addRecipient("c2"));
- frame.setMessage(put);
- replies.clear();
- replies.put("c0", ErrorCode.SERVICE_OOS);
- frame.assertMergeOk(replies, null);
-
- frame.setHop(new HopSpec("test", "[SearchColumn:2]").addRecipient("c0").addRecipient("c1").addRecipient("c2"));
- frame.setMessage(put);
- replies.clear();
- replies.put("c0", ErrorCode.SERVICE_OOS);
- frame.assertMergeOk(replies, null);
-
- frame.destroy();
- }
-
- private void assertDistribution(PolicyTestFrame frame, String id, String expected) {
- frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId(id)))));
- frame.assertSelect(Arrays.asList(expected));
- }
-
- @Test
- public void testSubsetService() {
- PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
- frame.setMessage(new PutDocumentMessage(new DocumentPut(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:"))))));
-
- // Test requerying for adding nodes.
- frame.setHop(new HopSpec("test", "docproc/cluster.default/[SubsetService:2]/chain.default"));
- Set<String> lst = new HashSet<>();
- for (int i = 1; i <= 10; ++i) {
- frame.getNetwork().registerSession(i + "/chain.default");
- assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", i));
-
- RoutingNode leaf = frame.select(1).get(0);
- lst.add(leaf.getRoute().toString());
- leaf.handleReply(new EmptyReply());
- assertNotNull(frame.getReceptor().getReply(TIMEOUT));
- }
- assertTrue(lst.size() > 1); // must have requeried
-
- // Test load balancing.
- String prev = null;
- for (int i = 1; i <= 10; ++i) {
- RoutingNode leaf = frame.select(1).get(0);
- String next = leaf.getRoute().toString();
- if (prev == null) {
- assertNotNull(next);
- } else {
- assertFalse(prev.equals(next));
- }
- prev = next;
- leaf.handleReply(new EmptyReply());
- assertNotNull(frame.getReceptor().getReply(TIMEOUT));
- }
-
- // Test requerying for dropping nodes.
- lst.clear();
- for (int i = 1; i <= 10; ++i) {
- RoutingNode leaf = frame.select(1).get(0);
- String route = leaf.getRoute().toString();
- lst.add(route);
-
- frame.getNetwork().unregisterSession(route.substring(frame.getIdentity().length() + 1));
- assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10 - i));
-
- Reply reply = new EmptyReply();
- reply.addError(new Error(ErrorCode.NO_ADDRESS_FOR_SERVICE, route));
- leaf.handleReply(reply);
- assertNotNull(frame.getReceptor().getReply(TIMEOUT));
- }
- assertEquals(10, lst.size());
-
- // Test merge behavior.
- frame.setHop(new HopSpec("test", "[SubsetService]"));
- frame.assertMergeOneReply("*");
-
- frame.destroy();
- }
-
- @Test
- public void testSubsetServiceCache() {
- PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);
- HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[SubsetService:2]/chain.foo");
- fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));
- fooFrame.setHop(fooHop);
-
- PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);
- HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[SubsetService:2]/chain.bar");
- barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));
- barFrame.setHop(barHop);
-
- fooFrame.getMessageBus().setupRouting(
- new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)
- .addHop(fooHop)
- .addHop(barHop)));
-
- fooFrame.getNetwork().registerSession("0/chain.foo");
- fooFrame.getNetwork().registerSession("0/chain.bar");
- assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));
-
- RoutingNode fooChild = fooFrame.select(1).get(0);
- assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString());
- RoutingNode barChild = barFrame.select(1).get(0);
- assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString());
-
- barChild.handleReply(new EmptyReply());
- fooChild.handleReply(new EmptyReply());
-
- assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));
- assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));
- }
-
- @Test
- public void testDocumentRouteSelector() {
- // Test policy usage safeguard.
- String okConfig = "raw:route[0]\n";
- String errConfig = "raw:" +
- "route[1]\n" +
- "route[0].name \"foo\"\n" +
- "route[0].selector \"foo bar\"\n" +
- "route[0].feed \"baz\"\n";
-
- DocumentProtocol protocol = new DocumentProtocol(manager, okConfig);
- assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof DocumentRouteSelectorPolicy);
- assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof DocumentRouteSelectorPolicy);
- assertTrue(protocol.createPolicy("DocumentRouteSelector", errConfig) instanceof ErrorPolicy);
-
- protocol = new DocumentProtocol(manager, errConfig);
- assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof ErrorPolicy);
- assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof ErrorPolicy);
- assertTrue(protocol.createPolicy("DocumentRouteSelector", okConfig) instanceof DocumentRouteSelectorPolicy);
-
- // Test policy with proper config.
- PolicyTestFrame frame = new PolicyTestFrame(manager);
- frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" +
- "route[2]\n" +
- "route[0].name \"foo\"\n" +
- "route[0].selector \"testdoc\"\n" +
- "route[0].feed \"myfeed\"\n" +
- "route[1].name \"bar\"\n" +
- "route[1].selector \"other\"\n" +
- "route[1].feed \"myfeed\"\n]").addRecipient("foo").addRecipient("bar"));
-
- frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:"), "fieldSet"));
- frame.assertSelect(Arrays.asList("bar", "foo"));
-
- Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:"))));
- frame.setMessage(put);
- frame.assertSelect(Arrays.asList("foo"));
-
- frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:")));
- frame.assertSelect(Arrays.asList("bar", "foo"));
-
- frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:"))));
- frame.assertSelect(Arrays.asList("foo"));
-
- frame.setMessage(put);
- frame.assertMergeOneReply("foo");
-
- frame.destroy();
- }
-
-
- @Test
- public void testDocumentRouteSelectorIgnore() {
- PolicyTestFrame frame = new PolicyTestFrame(manager);
- frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" +
- "route[1]\n" +
- "route[0].name \"docproc/cluster.foo\"\n" +
- "route[0].selector \"testdoc and testdoc.stringfield == 'foo'\"\n" +
- "route[0].feed \"myfeed\"\n]").addRecipient("docproc/cluster.foo"));
-
- frame.setMessage(new PutDocumentMessage(
- new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("id:yarn:testdoc:n=1234:fluff")))));
- frame.select(0);
- Reply reply = frame.getReceptor().getReply(TIMEOUT);
- assertNotNull(reply);
- assertEquals(DocumentProtocol.REPLY_DOCUMENTIGNORED, reply.getType());
- assertEquals(0, reply.getNumErrors());
-
- frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:"))));
- frame.assertSelect(Arrays.asList("docproc/cluster.foo"));
-
- frame.destroy();
- }
-
- @Test
- public void testLoadBalancer() {
- PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
- frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:")))));
- frame.getNetwork().registerSession("0/chain.default");
- assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 1));
- frame.setHop(new HopSpec("test", "[LoadBalancer:cluster=docproc/cluster.default;session=chain.default]"));
-
- assertSelect(frame, 1, Arrays.asList(frame.getNetwork().getConnectionSpec() + "/chain.default"));
- }
-
- @Test
- public void testRoundRobin() {
- // Test select with proper address.
- PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager);
- frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId("doc:scheme:")))));
- for (int i = 0; i < 10; ++i) {
- frame.getNetwork().registerSession(i + "/chain.default");
- }
- assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10));
- frame.setHop(new HopSpec("test", "[RoundRobin]")
- .addRecipient("docproc/cluster.default/3/chain.default")
- .addRecipient("docproc/cluster.default/6/chain.default")
- .addRecipient("docproc/cluster.default/9/chain.default"));
- assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default",
- "docproc/cluster.default/6/chain.default",
- "docproc/cluster.default/9/chain.default"));
- frame.getNetwork().unregisterSession("6/chain.default");
- assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 9));
- assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default",
- "docproc/cluster.default/9/chain.default"));
- frame.getNetwork().unregisterSession("3/chain.default");
- assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 8));
- assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/9/chain.default"));
- frame.getNetwork().unregisterSession("9/chain.default");
- assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 7));
- assertSelect(frame, 32, new ArrayList<String>());
-
- // Test merge behavior.
- frame.setHop(new HopSpec("test", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.default"));
- frame.assertMergeOneReply("docproc/cluster.default/0/chain.default");
-
- frame.destroy();
- }
-
- @Test
- public void testRoundRobinCache() {
- PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager);
- HopSpec fooHop = new HopSpec("foo", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.foo");
- fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo")));
- fooFrame.setHop(fooHop);
-
- PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame);
- HopSpec barHop = new HopSpec("bar", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.bar");
- barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar")));
- barFrame.setHop(barHop);
-
- fooFrame.getMessageBus().setupRouting(
- new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME)
- .addHop(fooHop)
- .addHop(barHop)));
-
- fooFrame.getNetwork().registerSession("0/chain.foo");
- fooFrame.getNetwork().registerSession("0/chain.bar");
- assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2));
-
- RoutingNode fooChild = fooFrame.select(1).get(0);
- assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().toString());
- RoutingNode barChild = barFrame.select(1).get(0);
- assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().toString());
-
- barChild.handleReply(new EmptyReply());
- fooChild.handleReply(new EmptyReply());
-
- assertNotNull(barFrame.getReceptor().getReply(TIMEOUT));
- assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT));
- }
-
- /**
- * Ensures that the given number of select passes on the given frame produces an expected list of recipients.
- *
- * @param frame The frame to select on.
- * @param numSelects The number of selects to perform.
- * @param expected The list of expected recipients.
- */
- private static void assertSelect(PolicyTestFrame frame, int numSelects, List<String> expected) {
- Set<String> lst = new TreeSet<>();
-
- for (int i = 0; i < numSelects; ++i) {
- if (!expected.isEmpty()) {
- RoutingNode leaf = frame.select(1).get(0);
- String recipient = leaf.getRoute().toString();
- lst.add(recipient);
- leaf.handleReply(new EmptyReply());
- } else {
- frame.select(0);
- }
- assertNotNull(frame.getReceptor().getReply(TIMEOUT));
- }
-
- assertEquals(expected.size(), lst.size());
- Iterator<String> it = lst.iterator();
- for (String recipient : expected) {
- assertEquals(recipient, it.next());
- }
- }
-
- private static void assertMirrorReady(Mirror slobrok)
- throws InterruptedException, TimeoutException
- {
- for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) {
- if (slobrok.ready()) {
- return;
- }
- Thread.sleep(10);
- }
- throw new TimeoutException();
- }
-
- private static void assertMirrorContains(IMirror slobrok, String pattern, int numEntries)
- throws InterruptedException, TimeoutException
- {
- for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) {
- if (slobrok.lookup(pattern).length == numEntries) {
- return;
- }
- Thread.sleep(10);
- }
- throw new TimeoutException();
- }
-
- private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern)
- throws InterruptedException, TimeoutException
- {
- setupExternPolicy(frame, slobrok, pattern, -1);
- }
-
- private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern, int numEntries)
- throws InterruptedException, TimeoutException
- {
- String param = "tcp/localhost:" + slobrok.port() + ";" + pattern;
- frame.setHop(new HopSpec("test", "[Extern:" + param + "]"));
- MessageBus mbus = frame.getMessageBus();
- HopBlueprint hop = mbus.getRoutingTable(DocumentProtocol.NAME).getHop("test");
- PolicyDirective dir = (PolicyDirective)hop.getDirective(0);
- ExternPolicy policy = (ExternPolicy)mbus.getRoutingPolicy(DocumentProtocol.NAME, dir.getName(), dir.getParam());
- assertMirrorReady(policy.getMirror());
- if (numEntries >= 0) {
- assertMirrorContains(policy.getMirror(), pattern, numEntries);
- }
- }
-
- private PolicyTestFrame newFrame() {
- return new PolicyTestFrame(manager);
- }
-
- private PolicyTestFrame newFrame(Message msg) {
- PolicyTestFrame frame = newFrame();
- frame.setMessage(msg);
- return frame;
- }
-
- private PutDocumentMessage newPutDocument(String documentId) {
- return new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"),
- new DocumentId(documentId))));
- }
-
- private PolicyTestFrame newPutDocumentFrame(String documentId) {
- return newFrame(newPutDocument(documentId));
- }
-}
+package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.document.*; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.slobrok.api.IMirror; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.jrt.slobrok.server.Slobrok; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.network.rpc.test.TestServer; +import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.test.Receptor; +import com.yahoo.vdslib.DocumentList; +import com.yahoo.vdslib.Entry; +import org.junit.Before; +import org.junit.Test; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@SuppressWarnings("deprecation") +public class PolicyTestCase { + + private static final int TIMEOUT = 300; + private static final TimeUnit TIMEOUT_UNIT = TimeUnit.SECONDS; + private static final long TIMEOUT_MILLIS = TIMEOUT_UNIT.toMillis(TIMEOUT); + private final DocumentTypeManager manager = new DocumentTypeManager(); + + @Before + public void setUp() { + DocumentTypeManagerConfigurer.configure(manager, "file:./test/cfg/testdoc.cfg"); + } + + @Test + public void testProtocol() { + DocumentProtocol protocol = new DocumentProtocol(manager); + + RoutingPolicy policy = protocol.createPolicy("AND", null); + assertTrue(policy instanceof ANDPolicy); + + policy = new DocumentProtocol(manager).createPolicy("DocumentRouteSelector", "raw:route[0]\n"); + assertTrue(policy instanceof DocumentRouteSelectorPolicy); + + policy = new DocumentProtocol(manager).createPolicy("Extern", "foo;bar/baz"); + assertTrue(policy instanceof ExternPolicy); + + policy = new DocumentProtocol(manager).createPolicy("LocalService", null); + assertTrue(policy instanceof LocalServicePolicy); + + policy = new DocumentProtocol(manager).createPolicy("RoundRobin", null); + assertTrue(policy instanceof RoundRobinPolicy); + + policy = new DocumentProtocol(manager).createPolicy("SearchRow", null); + assertTrue(policy instanceof SearchRowPolicy); + + policy = new DocumentProtocol(manager).createPolicy("SearchColumn", null); + assertTrue(policy instanceof SearchColumnPolicy); + + policy = new DocumentProtocol(manager).createPolicy("SubsetService", null); + assertTrue(policy instanceof SubsetServicePolicy); + + policy = new DocumentProtocol(manager).createPolicy("LoadBalancer", null); + assertTrue(policy instanceof LoadBalancerPolicy); + } + + @Test + public void testAND() { + PolicyTestFrame frame = new PolicyTestFrame(manager); + frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:"))))); + frame.setHop(new HopSpec("test", "[AND]") + .addRecipient("foo") + .addRecipient("bar")); + frame.assertSelect(Arrays.asList("foo", "bar")); + + frame.setHop(new HopSpec("test", "[AND:baz]") + .addRecipient("foo") + .addRecipient("bar")); + frame.assertSelect(Arrays.asList("baz")); // param precedes recipients + + frame.setHop(new HopSpec("test", "[AND:foo]")); + frame.assertMergeOneReply("foo"); + + frame.setHop(new HopSpec("test", "[AND:foo bar]")); + frame.assertMergeTwoReplies("foo", "bar"); + frame.destroy(); + } + + @Test + public void requireThatExternPolicyWithIllegalParamIsAnErrorPolicy() throws ListenFailedException { + Slobrok slobrok = new Slobrok(); + String spec = "tcp/localhost:" + slobrok.port(); + assertTrue(new DocumentProtocol(manager).createPolicy("Extern", null) instanceof ErrorPolicy); + assertTrue(new DocumentProtocol(manager).createPolicy("Extern", "") instanceof ErrorPolicy); + assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec) instanceof ErrorPolicy); + assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";") instanceof ErrorPolicy); + assertTrue(new DocumentProtocol(manager).createPolicy("Extern", spec + ";bar") instanceof ErrorPolicy); + } + + @Test + public void requireThatExternPolicyWithUnknownPatternSelectsNone() throws Exception { + PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:"); + setupExternPolicy(frame, new Slobrok(), "foo/bar"); + frame.assertSelect(null); + } + + @Test + public void requireThatExternPolicySelectsFromExternSlobrok() throws Exception { + PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:"); + Slobrok slobrok = new Slobrok(); + List<TestServer> servers = new ArrayList<>(); + for (int i = 0; i < 10; ++i) { + TestServer server = new TestServer("docproc/cluster.default/" + i, null, slobrok, null, + new DocumentProtocol(manager)); + server.net.registerSession("chain.default"); + servers.add(server); + } + setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 10); + Set<String> lst = new HashSet<>(); + for (int i = 0; i < 10; ++i) { + RoutingNode leaf = frame.select(1).get(0); + String recipient = leaf.getRoute().toString(); + lst.add(recipient); + + leaf.handleReply(new EmptyReply()); + assertNotNull(frame.getReceptor().getReply(TIMEOUT)); + } + assertEquals(10, lst.size()); + for (TestServer server : servers) { + server.destroy(); + } + frame.destroy(); + } + + @Test + public void requireThatExternPolicyMergesOneReplyAsProtocol() throws Exception { + PolicyTestFrame frame = newPutDocumentFrame("doc:scheme:"); + Slobrok slobrok = new Slobrok(); + TestServer server = new TestServer("docproc/cluster.default/0", null, slobrok, null, + new DocumentProtocol(manager)); + server.net.registerSession("chain.default"); + setupExternPolicy(frame, slobrok, "docproc/cluster.default/*/chain.default", 1); + frame.assertMergeOneReply(server.net.getConnectionSpec() + "/chain.default"); + server.destroy(); + frame.destroy(); + } + + @Test + public void testExternSend() throws Exception { + // Setup local source node. + Slobrok local = new Slobrok(); + TestServer src = new TestServer("src", null, local, null, new DocumentProtocol(manager)); + SourceSession ss = src.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT)); + + // Setup remote cluster with routing config. + Slobrok slobrok = new Slobrok(); + TestServer itr = new TestServer("itr", + new RoutingTableSpec(DocumentProtocol.NAME) + .addRoute(new RouteSpec("default").addHop("dst")) + .addHop(new HopSpec("dst", "dst/session")), + slobrok, null, new DocumentProtocol(manager)); + IntermediateSession is = itr.mb.createIntermediateSession("session", true, new Receptor(), new Receptor()); + TestServer dst = new TestServer("dst", null, slobrok, null, new DocumentProtocol(manager)); + DestinationSession ds = dst.mb.createDestinationSession("session", true, new Receptor()); + + // Send message from local node to remote cluster and resolve route there. + Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); + msg.getTrace().setLevel(9); + msg.setRoute(Route.parse("[Extern:tcp/localhost:" + slobrok.port() + ";itr/session] default")); + + assertTrue(ss.send(msg).isAccepted()); + assertNotNull(msg = ((Receptor)is.getMessageHandler()).getMessage(TIMEOUT)); + is.forward(msg); + assertNotNull(msg = ((Receptor)ds.getMessageHandler()).getMessage(TIMEOUT)); + ds.acknowledge(msg); + Reply reply = ((Receptor)is.getReplyHandler()).getReply(TIMEOUT); + assertNotNull(reply); + is.forward(reply); + assertNotNull(reply = ((Receptor)ss.getReplyHandler()).getReply(TIMEOUT)); + + System.out.println(reply.getTrace().toString()); + + // Perform necessary cleanup. + src.destroy(); + itr.destroy(); + dst.destroy(); + slobrok.stop(); + local.stop(); + } + + @Test + public void testExternMultipleSlobroks() throws ListenFailedException { + Slobrok local = new Slobrok(); + TestServer srcServer = new TestServer("src", null, local, null, new DocumentProtocol(manager)); + SourceSession srcSession = + srcServer.mb.createSourceSession(new Receptor(), new SourceSessionParams().setTimeout(TIMEOUT)); + + Slobrok extern = new Slobrok(); + String spec = "tcp/localhost:" + extern.port(); + + TestServer dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager)); + Receptor dstHandler = new Receptor(); + DestinationSession dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler); + + Message msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); + msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]")); + assertTrue(srcSession.send(msg).isAccepted()); + assertNotNull(msg = dstHandler.getMessage(TIMEOUT)); + dstSession.acknowledge(msg); + Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT); + assertNotNull(reply); + + extern.stop(); + dstSession.destroy(); + dstServer.destroy(); + dstHandler.reset(); + assertNull(dstHandler.getMessage(0)); + + extern = new Slobrok(); + spec += ",tcp/localhost:" + extern.port(); + + dstServer = new TestServer("dst", null, extern, null, new DocumentProtocol(manager)); + dstHandler = new Receptor(); + dstSession = dstServer.mb.createDestinationSession("session", true, dstHandler); + + msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); + msg.setRoute(Route.parse("[Extern:" + spec + ";dst/session]")); + assertTrue(srcSession.send(msg).isAccepted()); + assertNotNull(msg = dstHandler.getMessage(TIMEOUT)); + dstSession.acknowledge(msg); + reply = ((Receptor)srcSession.getReplyHandler()).getReply(TIMEOUT); + assertNotNull(reply); + + extern.stop(); + dstSession.destroy(); + dstServer.destroy(); + + local.stop(); + srcSession.destroy(); + srcServer.destroy(); + } + + @Test + public void testLocalService() { + // Test select with proper address. + PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager); + frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:0"))))); + for (int i = 0; i < 10; ++i) { + frame.getNetwork().registerSession(i + "/chain.default"); + } + assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10)); + frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService]/chain.default")); + + Set<String> lst = new HashSet<>(); + for (int i = 0; i < 10; ++i) { + RoutingNode leaf = frame.select(1).get(0); + String recipient = leaf.getRoute().toString(); + lst.add(recipient); + + leaf.handleReply(new EmptyReply()); + assertNotNull(frame.getReceptor().getReply(TIMEOUT)); + } + assertEquals(10, lst.size()); + + // Test select with broken address. + lst.clear(); + frame.setHop(new HopSpec("test", "docproc/cluster.default/[LocalService:broken]/chain.default")); + for (int i = 0; i < 10; ++i) { + RoutingNode leaf = frame.select(1).get(0); + String recipient = leaf.getRoute().toString(); + assertTrue(recipient.equals("docproc/cluster.default/*/chain.default")); + lst.add(recipient); + + leaf.handleReply(new EmptyReply()); + assertNotNull(frame.getReceptor().getReply(TIMEOUT)); + } + assertEquals(1, lst.size()); + + // Test merge behavior. + frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:"))))); + frame.setHop(new HopSpec("test", "[LocalService]")); + frame.assertMergeOneReply("*"); + + frame.destroy(); + } + + @Test + public void testLocalServiceCache() { + PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager); + HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[LocalService]/chain.foo"); + fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo"))); + fooFrame.setHop(fooHop); + + PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame); + HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[LocalService]/chain.bar"); + barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar"))); + barFrame.setHop(barHop); + + fooFrame.getMessageBus().setupRouting( + new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME) + .addHop(fooHop) + .addHop(barHop))); + + fooFrame.getNetwork().registerSession("0/chain.foo"); + fooFrame.getNetwork().registerSession("0/chain.bar"); + assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2)); + + RoutingNode fooChild = fooFrame.select(1).get(0); + assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString()); + RoutingNode barChild = barFrame.select(1).get(0); + assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString()); + + barChild.handleReply(new EmptyReply()); + fooChild.handleReply(new EmptyReply()); + + assertNotNull(barFrame.getReceptor().getReply(TIMEOUT)); + assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT)); + } + + @Test + public void testSearchRow() { + PolicyTestFrame frame = new PolicyTestFrame(manager); + frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:"))))); + frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo")); + frame.assertMergeOneReply("foo"); + frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar")); + frame.assertMergeTwoReplies("foo", "bar"); + + frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo")); + Map<String, Integer> replies = new HashMap<>(); + replies.put("foo", ErrorCode.SERVICE_OOS); + frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS)); + + frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar")); + replies.put("foo", ErrorCode.SERVICE_OOS); + replies.put("bar", ErrorCode.NONE); + frame.assertMergeOk(replies, Arrays.asList("bar")); + + replies.put("foo", ErrorCode.SERVICE_OOS); + replies.put("bar", ErrorCode.SERVICE_OOS); + frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS)); + + frame.setHop(new HopSpec("test", "[SearchRow:1]").addRecipient("foo").addRecipient("bar").addRecipient("baz")); + replies.put("foo", ErrorCode.SERVICE_OOS); + replies.put("bar", ErrorCode.NONE); + replies.put("baz", ErrorCode.NONE); + frame.assertMergeOk(replies, Arrays.asList("bar", "baz")); + + replies.put("foo", ErrorCode.SERVICE_OOS); + replies.put("bar", ErrorCode.SERVICE_OOS); + replies.put("baz", ErrorCode.NONE); + frame.assertMergeOk(replies, Arrays.asList("baz")); + + replies.put("foo", ErrorCode.SERVICE_OOS); + replies.put("bar", ErrorCode.SERVICE_OOS); + replies.put("baz", ErrorCode.SERVICE_OOS); + frame.assertMergeError(replies, + Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS)); + + frame.setHop(new HopSpec("test", "[SearchRow:2]").addRecipient("foo").addRecipient("bar").addRecipient("baz")); + replies.put("foo", ErrorCode.SERVICE_OOS); + replies.put("bar", ErrorCode.NONE); + replies.put("baz", ErrorCode.NONE); + frame.assertMergeOk(replies, Arrays.asList("bar", "baz")); + + replies.put("foo", ErrorCode.SERVICE_OOS); + replies.put("bar", ErrorCode.SERVICE_OOS); + replies.put("baz", ErrorCode.NONE); + frame.assertMergeError(replies, Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS)); + + replies.put("foo", ErrorCode.SERVICE_OOS); + replies.put("bar", ErrorCode.SERVICE_OOS); + replies.put("baz", ErrorCode.SERVICE_OOS); + frame.assertMergeError(replies, + Arrays.asList(ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS, ErrorCode.SERVICE_OOS)); + + frame.destroy(); + } + + @Test + public void testSearchRowMerge() { + PolicyTestFrame frame = new PolicyTestFrame(manager); + frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo")); + tryWasFound(frame, 1, 0x0, false); + tryWasFound(frame, 1, 0x1, true); + + frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar")); + tryWasFound(frame, 2, 0x0, false); + tryWasFound(frame, 2, 0x1, true); + tryWasFound(frame, 2, 0x2, true); + tryWasFound(frame, 2, 0x3, true); + + frame.setHop(new HopSpec("test", "[SearchRow]").addRecipient("foo").addRecipient("bar").addRecipient("baz")); + tryWasFound(frame, 3, 0x0, false); + tryWasFound(frame, 3, 0x1, true); + tryWasFound(frame, 3, 0x2, true); + tryWasFound(frame, 3, 0x3, true); + tryWasFound(frame, 3, 0x4, true); + tryWasFound(frame, 3, 0x5, true); + tryWasFound(frame, 3, 0x6, true); + tryWasFound(frame, 3, 0x7, true); + frame.destroy(); + } + + private void tryWasFound(PolicyTestFrame frame, int expectedRecipients, + int foundMask, boolean expectedFound) + { + { + frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:69"))); + List<RoutingNode> selected = frame.select(expectedRecipients); + for (int i = 0, len = selected.size(); i < len; ++i) { + RemoveDocumentReply reply = new RemoveDocumentReply(); + reply.setWasFound(((1 << i) & foundMask) != 0); + selected.get(i).handleReply(reply); + } + Reply reply = frame.getReceptor().getReply(TIMEOUT); + assertNotNull(reply); + assertEquals(DocumentProtocol.REPLY_REMOVEDOCUMENT, reply.getType()); + assertEquals(expectedFound, ((RemoveDocumentReply)reply).wasFound()); + } + { + DocumentUpdate upd = new DocumentUpdate(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:")); + frame.setMessage(new UpdateDocumentMessage(upd)); + List<RoutingNode> selected = frame.select(expectedRecipients); + for (int i = 0, len = selected.size(); i < len; ++i) { + UpdateDocumentReply reply = new UpdateDocumentReply(); + reply.setWasFound(((1 << i) & foundMask) != 0); + selected.get(i).handleReply(reply); + } + Reply reply = frame.getReceptor().getReply(TIMEOUT); + assertNotNull(reply); + assertEquals(DocumentProtocol.REPLY_UPDATEDOCUMENT, reply.getType()); + assertEquals(expectedFound, ((UpdateDocumentReply)reply).wasFound()); + } + } + + @Test + public void multipleGetRepliesAreMergedToFoundDocument() { + PolicyTestFrame frame = new PolicyTestFrame(manager); + frame.setHop(new HopSpec("test", getDocumentRouteSelectorRawConfig()) + .addRecipient("foo").addRecipient("bar")); + frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:yarn"), "[all]")); + List<RoutingNode> selected = frame.select(2); + for (int i = 0, len = selected.size(); i < len; ++i) { + Document doc = null; + if (i == 0) { + doc = new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:yarn")); + doc.setLastModified(123456L); + } + GetDocumentReply reply = new GetDocumentReply(null); + reply.setDocument(doc); + selected.get(i).handleReply(reply); + } + Reply reply = frame.getReceptor().getReply(TIMEOUT); + assertNotNull(reply); + assertEquals(DocumentProtocol.REPLY_GETDOCUMENT, reply.getType()); + assertEquals(123456L, ((GetDocumentReply)reply).getLastModified()); + } + + private String getDocumentRouteSelectorRawConfig() { + return "[DocumentRouteSelector:raw:" + + "route[2]\n" + + "route[0].name \"foo\"\n" + + "route[0].selector \"testdoc\"\n" + + "route[0].feed \"myfeed\"\n" + + "route[1].name \"bar\"\n" + + "route[1].selector \"other\"\n" + + "route[1].feed \"myfeed\"\n]"; + } + + @Test + public void testSearchColumn() { + PolicyTestFrame frame = new PolicyTestFrame(manager); + frame.setHop(new HopSpec("test", "[SearchColumn]") + .addRecipient("c0").addRecipient("c1") + .addRecipient("c2").addRecipient("c3")); + + // Test hash distribution. + assertDistribution(frame, "doc:ns:3", "c0"); + assertDistribution(frame, "doc:ns:18", "c1"); + assertDistribution(frame, "doc:ns:0", "c2"); + assertDistribution(frame, "doc:ns:4", "c3"); + + assertDistribution(frame, "userdoc:ns:49152:0", "c0"); + assertDistribution(frame, "userdoc:ns:49152:1", "c0"); + assertDistribution(frame, "userdoc:ns:16384:2", "c1"); + assertDistribution(frame, "userdoc:ns:16384:3", "c1"); + assertDistribution(frame, "userdoc:ns:5461:4", "c2"); + assertDistribution(frame, "userdoc:ns:5461:5", "c2"); + assertDistribution(frame, "userdoc:ns:0:6", "c3"); + assertDistribution(frame, "userdoc:ns:0:7", "c3"); + + assertDistribution(frame, "groupdoc:ns:0:0", "c0"); + assertDistribution(frame, "groupdoc:ns:0:1", "c0"); + assertDistribution(frame, "groupdoc:ns:4:2", "c1"); + assertDistribution(frame, "groupdoc:ns:4:3", "c1"); + assertDistribution(frame, "groupdoc:ns:2:4", "c2"); + assertDistribution(frame, "groupdoc:ns:2:5", "c2"); + assertDistribution(frame, "groupdoc:ns:7:6", "c3"); + assertDistribution(frame, "groupdoc:ns:7:7", "c3"); + + // Test routing based on message type. + Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:")))); + frame.setHop(new HopSpec("test", "[SearchColumn]").addRecipient("c0").addRecipient("c1")); + frame.setMessage(put); + frame.assertMergeOneReply("c0"); + + // Test allowed bad parts. + frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0")); + frame.setMessage(put); + Map<String, Integer> replies = new HashMap<>(); + replies.put("c0", ErrorCode.SERVICE_OOS); + frame.assertMergeOk(replies, null); + + replies.put("c0", ErrorCode.SERVICE_OOS); + frame.assertMergeOk(replies, null); + + frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1")); + frame.setMessage(put); + replies.put("c0", ErrorCode.SERVICE_OOS); + frame.assertMergeOk(replies, null); + + frame.setHop(new HopSpec("test", "[SearchColumn:1]").addRecipient("c0").addRecipient("c1").addRecipient("c2")); + frame.setMessage(put); + replies.clear(); + replies.put("c0", ErrorCode.SERVICE_OOS); + frame.assertMergeOk(replies, null); + + frame.setHop(new HopSpec("test", "[SearchColumn:2]").addRecipient("c0").addRecipient("c1").addRecipient("c2")); + frame.setMessage(put); + replies.clear(); + replies.put("c0", ErrorCode.SERVICE_OOS); + frame.assertMergeOk(replies, null); + + frame.destroy(); + } + + private void assertDistribution(PolicyTestFrame frame, String id, String expected) { + frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId(id))))); + frame.assertSelect(Arrays.asList(expected)); + } + + @Test + public void testSubsetService() { + PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager); + frame.setMessage(new PutDocumentMessage(new DocumentPut(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:")))))); + + // Test requerying for adding nodes. + frame.setHop(new HopSpec("test", "docproc/cluster.default/[SubsetService:2]/chain.default")); + Set<String> lst = new HashSet<>(); + for (int i = 1; i <= 10; ++i) { + frame.getNetwork().registerSession(i + "/chain.default"); + assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", i)); + + RoutingNode leaf = frame.select(1).get(0); + lst.add(leaf.getRoute().toString()); + leaf.handleReply(new EmptyReply()); + assertNotNull(frame.getReceptor().getReply(TIMEOUT)); + } + assertTrue(lst.size() > 1); // must have requeried + + // Test load balancing. + String prev = null; + for (int i = 1; i <= 10; ++i) { + RoutingNode leaf = frame.select(1).get(0); + String next = leaf.getRoute().toString(); + if (prev == null) { + assertNotNull(next); + } else { + assertFalse(prev.equals(next)); + } + prev = next; + leaf.handleReply(new EmptyReply()); + assertNotNull(frame.getReceptor().getReply(TIMEOUT)); + } + + // Test requerying for dropping nodes. + lst.clear(); + for (int i = 1; i <= 10; ++i) { + RoutingNode leaf = frame.select(1).get(0); + String route = leaf.getRoute().toString(); + lst.add(route); + + frame.getNetwork().unregisterSession(route.substring(frame.getIdentity().length() + 1)); + assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10 - i)); + + Reply reply = new EmptyReply(); + reply.addError(new Error(ErrorCode.NO_ADDRESS_FOR_SERVICE, route)); + leaf.handleReply(reply); + assertNotNull(frame.getReceptor().getReply(TIMEOUT)); + } + assertEquals(10, lst.size()); + + // Test merge behavior. + frame.setHop(new HopSpec("test", "[SubsetService]")); + frame.assertMergeOneReply("*"); + + frame.destroy(); + } + + @Test + public void testSubsetServiceCache() { + PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager); + HopSpec fooHop = new HopSpec("foo", "docproc/cluster.default/[SubsetService:2]/chain.foo"); + fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo"))); + fooFrame.setHop(fooHop); + + PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame); + HopSpec barHop = new HopSpec("bar", "docproc/cluster.default/[SubsetService:2]/chain.bar"); + barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar"))); + barFrame.setHop(barHop); + + fooFrame.getMessageBus().setupRouting( + new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME) + .addHop(fooHop) + .addHop(barHop))); + + fooFrame.getNetwork().registerSession("0/chain.foo"); + fooFrame.getNetwork().registerSession("0/chain.bar"); + assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2)); + + RoutingNode fooChild = fooFrame.select(1).get(0); + assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().getHop(0).toString()); + RoutingNode barChild = barFrame.select(1).get(0); + assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().getHop(0).toString()); + + barChild.handleReply(new EmptyReply()); + fooChild.handleReply(new EmptyReply()); + + assertNotNull(barFrame.getReceptor().getReply(TIMEOUT)); + assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT)); + } + + @Test + public void testDocumentRouteSelector() { + // Test policy usage safeguard. + String okConfig = "raw:route[0]\n"; + String errConfig = "raw:" + + "route[1]\n" + + "route[0].name \"foo\"\n" + + "route[0].selector \"foo bar\"\n" + + "route[0].feed \"baz\"\n"; + + DocumentProtocol protocol = new DocumentProtocol(manager, okConfig); + assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof DocumentRouteSelectorPolicy); + assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof DocumentRouteSelectorPolicy); + assertTrue(protocol.createPolicy("DocumentRouteSelector", errConfig) instanceof ErrorPolicy); + + protocol = new DocumentProtocol(manager, errConfig); + assertTrue(protocol.createPolicy("DocumentRouteSelector", null) instanceof ErrorPolicy); + assertTrue(protocol.createPolicy("DocumentRouteSelector", "") instanceof ErrorPolicy); + assertTrue(protocol.createPolicy("DocumentRouteSelector", okConfig) instanceof DocumentRouteSelectorPolicy); + + // Test policy with proper config. + PolicyTestFrame frame = new PolicyTestFrame(manager); + frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" + + "route[2]\n" + + "route[0].name \"foo\"\n" + + "route[0].selector \"testdoc\"\n" + + "route[0].feed \"myfeed\"\n" + + "route[1].name \"bar\"\n" + + "route[1].selector \"other\"\n" + + "route[1].feed \"myfeed\"\n]").addRecipient("foo").addRecipient("bar")); + + frame.setMessage(new GetDocumentMessage(new DocumentId("doc:scheme:"), "fieldSet")); + frame.assertSelect(Arrays.asList("bar", "foo")); + + Message put = new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:")))); + frame.setMessage(put); + frame.assertSelect(Arrays.asList("foo")); + + frame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:"))); + frame.assertSelect(Arrays.asList("bar", "foo")); + + frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:")))); + frame.assertSelect(Arrays.asList("foo")); + + frame.setMessage(put); + frame.assertMergeOneReply("foo"); + + frame.destroy(); + } + + + @Test + public void testDocumentRouteSelectorIgnore() { + PolicyTestFrame frame = new PolicyTestFrame(manager); + frame.setHop(new HopSpec("test", "[DocumentRouteSelector:raw:" + + "route[1]\n" + + "route[0].name \"docproc/cluster.foo\"\n" + + "route[0].selector \"testdoc and testdoc.stringfield == 'foo'\"\n" + + "route[0].feed \"myfeed\"\n]").addRecipient("docproc/cluster.foo")); + + frame.setMessage(new PutDocumentMessage( + new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("id:yarn:testdoc:n=1234:fluff"))))); + frame.select(0); + Reply reply = frame.getReceptor().getReply(TIMEOUT); + assertNotNull(reply); + assertEquals(DocumentProtocol.REPLY_DOCUMENTIGNORED, reply.getType()); + assertEquals(0, reply.getNumErrors()); + + frame.setMessage(new UpdateDocumentMessage(new DocumentUpdate(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:")))); + frame.assertSelect(Arrays.asList("docproc/cluster.foo")); + + frame.destroy(); + } + + @Test + public void testLoadBalancer() { + PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager); + frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:"))))); + frame.getNetwork().registerSession("0/chain.default"); + assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 1)); + frame.setHop(new HopSpec("test", "[LoadBalancer:cluster=docproc/cluster.default;session=chain.default]")); + + assertSelect(frame, 1, Arrays.asList(frame.getNetwork().getConnectionSpec() + "/chain.default")); + } + + @Test + public void testRoundRobin() { + // Test select with proper address. + PolicyTestFrame frame = new PolicyTestFrame("docproc/cluster.default", manager); + frame.setMessage(new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId("doc:scheme:"))))); + for (int i = 0; i < 10; ++i) { + frame.getNetwork().registerSession(i + "/chain.default"); + } + assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 10)); + frame.setHop(new HopSpec("test", "[RoundRobin]") + .addRecipient("docproc/cluster.default/3/chain.default") + .addRecipient("docproc/cluster.default/6/chain.default") + .addRecipient("docproc/cluster.default/9/chain.default")); + assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default", + "docproc/cluster.default/6/chain.default", + "docproc/cluster.default/9/chain.default")); + frame.getNetwork().unregisterSession("6/chain.default"); + assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 9)); + assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/3/chain.default", + "docproc/cluster.default/9/chain.default")); + frame.getNetwork().unregisterSession("3/chain.default"); + assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 8)); + assertSelect(frame, 32, Arrays.asList("docproc/cluster.default/9/chain.default")); + frame.getNetwork().unregisterSession("9/chain.default"); + assertTrue(frame.waitSlobrok("docproc/cluster.default/*/chain.default", 7)); + assertSelect(frame, 32, new ArrayList<String>()); + + // Test merge behavior. + frame.setHop(new HopSpec("test", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.default")); + frame.assertMergeOneReply("docproc/cluster.default/0/chain.default"); + + frame.destroy(); + } + + @Test + public void testRoundRobinCache() { + PolicyTestFrame fooFrame = new PolicyTestFrame("docproc/cluster.default", manager); + HopSpec fooHop = new HopSpec("foo", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.foo"); + fooFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:foo"))); + fooFrame.setHop(fooHop); + + PolicyTestFrame barFrame = new PolicyTestFrame(fooFrame); + HopSpec barHop = new HopSpec("bar", "[RoundRobin]").addRecipient("docproc/cluster.default/0/chain.bar"); + barFrame.setMessage(new RemoveDocumentMessage(new DocumentId("doc:scheme:bar"))); + barFrame.setHop(barHop); + + fooFrame.getMessageBus().setupRouting( + new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME) + .addHop(fooHop) + .addHop(barHop))); + + fooFrame.getNetwork().registerSession("0/chain.foo"); + fooFrame.getNetwork().registerSession("0/chain.bar"); + assertTrue(fooFrame.waitSlobrok("docproc/cluster.default/0/*", 2)); + + RoutingNode fooChild = fooFrame.select(1).get(0); + assertEquals("docproc/cluster.default/0/chain.foo", fooChild.getRoute().toString()); + RoutingNode barChild = barFrame.select(1).get(0); + assertEquals("docproc/cluster.default/0/chain.bar", barChild.getRoute().toString()); + + barChild.handleReply(new EmptyReply()); + fooChild.handleReply(new EmptyReply()); + + assertNotNull(barFrame.getReceptor().getReply(TIMEOUT)); + assertNotNull(fooFrame.getReceptor().getReply(TIMEOUT)); + } + + /** + * Ensures that the given number of select passes on the given frame produces an expected list of recipients. + * + * @param frame The frame to select on. + * @param numSelects The number of selects to perform. + * @param expected The list of expected recipients. + */ + private static void assertSelect(PolicyTestFrame frame, int numSelects, List<String> expected) { + Set<String> lst = new TreeSet<>(); + + for (int i = 0; i < numSelects; ++i) { + if (!expected.isEmpty()) { + RoutingNode leaf = frame.select(1).get(0); + String recipient = leaf.getRoute().toString(); + lst.add(recipient); + leaf.handleReply(new EmptyReply()); + } else { + frame.select(0); + } + assertNotNull(frame.getReceptor().getReply(TIMEOUT)); + } + + assertEquals(expected.size(), lst.size()); + Iterator<String> it = lst.iterator(); + for (String recipient : expected) { + assertEquals(recipient, it.next()); + } + } + + private static void assertMirrorReady(Mirror slobrok) + throws InterruptedException, TimeoutException + { + for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) { + if (slobrok.ready()) { + return; + } + Thread.sleep(10); + } + throw new TimeoutException(); + } + + private static void assertMirrorContains(IMirror slobrok, String pattern, int numEntries) + throws InterruptedException, TimeoutException + { + for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) { + if (slobrok.lookup(pattern).length == numEntries) { + return; + } + Thread.sleep(10); + } + throw new TimeoutException(); + } + + private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern) + throws InterruptedException, TimeoutException + { + setupExternPolicy(frame, slobrok, pattern, -1); + } + + private void setupExternPolicy(PolicyTestFrame frame, Slobrok slobrok, String pattern, int numEntries) + throws InterruptedException, TimeoutException + { + String param = "tcp/localhost:" + slobrok.port() + ";" + pattern; + frame.setHop(new HopSpec("test", "[Extern:" + param + "]")); + MessageBus mbus = frame.getMessageBus(); + HopBlueprint hop = mbus.getRoutingTable(DocumentProtocol.NAME).getHop("test"); + PolicyDirective dir = (PolicyDirective)hop.getDirective(0); + ExternPolicy policy = (ExternPolicy)mbus.getRoutingPolicy(DocumentProtocol.NAME, dir.getName(), dir.getParam()); + assertMirrorReady(policy.getMirror()); + if (numEntries >= 0) { + assertMirrorContains(policy.getMirror(), pattern, numEntries); + } + } + + private PolicyTestFrame newFrame() { + return new PolicyTestFrame(manager); + } + + private PolicyTestFrame newFrame(Message msg) { + PolicyTestFrame frame = newFrame(); + frame.setMessage(msg); + return frame; + } + + private PutDocumentMessage newPutDocument(String documentId) { + return new PutDocumentMessage(new DocumentPut(new Document(manager.getDocumentType("testdoc"), + new DocumentId(documentId)))); + } + + private PolicyTestFrame newPutDocumentFrame(String documentId) { + return newFrame(newPutDocument(documentId)); + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java index 37b6c1a9068..f3721ff0173 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java @@ -1,385 +1,385 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.jrt.ListenFailedException;
-import com.yahoo.jrt.slobrok.api.Mirror;
-import com.yahoo.jrt.slobrok.server.Slobrok;
-import com.yahoo.messagebus.*;
-import com.yahoo.messagebus.network.Identity;
-import com.yahoo.messagebus.network.Network;
-import com.yahoo.messagebus.network.ServiceAddress;
-import com.yahoo.messagebus.network.rpc.RPCNetwork;
-import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
-import com.yahoo.messagebus.network.rpc.test.TestServer;
-import com.yahoo.messagebus.routing.*;
-import com.yahoo.messagebus.test.Receptor;
-import com.yahoo.messagebus.test.SimpleProtocol;
-import com.yahoo.messagebus.test.SimpleReply;
-
-import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * This is a utility class to allow easier policy test cases. The most important reason to use this is to make sure that
- * each test uses a "clean" mbus and slobrok instance.
- *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class PolicyTestFrame extends junit.framework.Assert {
-
- private final AtomicBoolean destroyed = new AtomicBoolean(false);
- private String identity;
- private Slobrok slobrok;
- private MessageBus mbus;
- private MyNetwork net;
- private Message msg = null;
- private HopSpec hop = null;
- private Receptor handler = new Receptor();
-
- /**
- * Create an anonymous test frame.
- *
- * @param documentMgr The document manager to use.
- */
- public PolicyTestFrame(DocumentTypeManager documentMgr) {
- this("anonymous", documentMgr);
- }
-
- /**
- * Create a named test frame.
- *
- * @param identity The identity to use for the server.
- * @param documentMgr The document manager to use.
- */
- public PolicyTestFrame(String identity, DocumentTypeManager documentMgr) {
- this.identity = identity;
- try {
- slobrok = new Slobrok();
- } catch (ListenFailedException e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- net = new MyNetwork(new RPCNetworkParams()
- .setIdentity(new Identity(identity))
- .setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
- mbus = new MessageBus(net, new MessageBusParams()
- .addProtocol(new DocumentProtocol(documentMgr)));
- }
-
- /**
- * Create a test frame running on the same slobrok and mbus as another.
- *
- * @param frame The frame whose internals to share.
- */
- public PolicyTestFrame(PolicyTestFrame frame) {
- identity = frame.identity;
- slobrok = frame.slobrok;
- net = frame.net;
- mbus = frame.mbus;
- }
-
- // Inherit doc from Object.
- @Override
- public void finalize() throws Throwable {
- destroy();
- super.finalize();
- }
-
- /**
- * Sets the destroyed flag to true. The very first time this method is called, it cleans up all its dependencies.
- * Even if you retain a reference to this object, all of its content is allowed to be garbage collected.
- */
- public void destroy() {
- if (!destroyed.getAndSet(true)) {
- mbus.destroy();
- mbus = null;
- net = null;
-
- slobrok.stop();
- slobrok = null;
- }
- }
-
- /**
- * Routes the contained message based on the current setup, and returns the leaf send contexts.
- *
- * @param numExpected The expected number of leaf nodes.
- * @return The list of selected send contexts.
- */
- public List<RoutingNode> select(int numExpected) {
- msg.setRoute(Route.parse(hop.getName()));
- new RoutingNode(mbus, net, null, handler, msg).send();
- List<RoutingNode> ret = net.removeNodes();
- assertEquals(numExpected, ret.size());
- return ret;
- }
-
- /**
- * Ensures that the current setup selects a given set of routes.
- *
- * @param expected A list of expected route leaf nodes.
- */
- public void assertSelect(List<String> expected) {
- if (expected == null) {
- assertEquals(0, select(0).size());
- } else {
- List<RoutingNode> selected = select(expected.size());
- for (RoutingNode node : selected) {
- assertTrue("Route '" + node.getRoute() + "' not selected.",
- expected.contains(node.getRoute().toString()));
- node.handleReply(new EmptyReply());
- }
- }
- assertNotNull(handler.getReply(60));
- }
-
- /**
- * This is a convenience method for invoking {@link #assertMerge(Map,List,List)} with no expected value.
- *
- * @param replies The errors to set in the leaf node replies.
- * @param expectedErrors The list of expected errors in the merged reply.
- */
- public void assertMergeError(Map<String, Integer> replies, List<Integer> expectedErrors) {
- assertMerge(replies, expectedErrors, null);
- }
-
- /**
- * This is a convenience method for invoking {@link this#assertMerge(Map,List,List)} with no expected errors.
- *
- * @param replies The errors to set in the leaf node replies.
- * @param allowedValues The list of allowed values in the final reply.
- */
- public void assertMergeOk(Map<String, Integer> replies, List<String> allowedValues) {
- assertMerge(replies, null, allowedValues);
- }
-
- /**
- * Ensures that the current setup generates as many leaf nodes as there are members of the errors argument. Each
- * error is then given one of these errors, and the method then ensures that the single returned reply contains the
- * given list of expected errors. Finally, if the expected value argument is non-null, this method ensures that the
- * reply is a SimpleReply whose string value exists in the allowed list.
- *
- * @param replies The errors to set in the leaf node replies.
- * @param expectedErrors The list of expected errors in the merged reply.
- * @param allowedValues The list of allowed values in the final reply.
- */
- public void assertMerge(Map<String, Integer> replies, List<Integer> expectedErrors, List<String> allowedValues) {
- List<RoutingNode> selected = select(replies.size());
-
- for (RoutingNode node : selected) {
- String route = node.getRoute().toString();
- assertTrue(replies.containsKey(route));
- Reply ret = new SimpleReply(route);
- if (replies.get(route) != ErrorCode.NONE) {
- ret.addError(new com.yahoo.messagebus.Error(replies.get(route), route));
- }
- node.handleReply(ret);
- }
-
- Reply reply = handler.getReply(60);
- assertNotNull(reply);
- if (expectedErrors != null) {
- assertEquals(expectedErrors.size(), reply.getNumErrors());
- for (int i = 0; i < expectedErrors.size(); ++i) {
- assertTrue(expectedErrors.contains(reply.getError(i).getCode()));
- }
- } else if (reply.hasErrors()) {
- StringBuilder err = new StringBuilder("Got unexpected error(s):\n");
- for (int i = 0; i < reply.getNumErrors(); ++i) {
- err.append("\t").append(reply.getError(i).toString());
- if (i < reply.getNumErrors() - 1) {
- err.append("\n");
- }
- }
- fail(err.toString());
- }
- if (allowedValues != null) {
- assertEquals(SimpleProtocol.REPLY, reply.getType());
- assertTrue(allowedValues.contains(((SimpleReply)reply).getValue()));
- } else {
- assertEquals(0, reply.getType());
- }
- }
-
- /**
- * Ensures that the current setup chooses a single recipient, and that it merges similarly to how the
- * {@link DocumentProtocol} would merge these.
- *
- * @param recipient The expected recipient.
- */
- public void assertMergeOneReply(String recipient) {
- assertSelect(Arrays.asList(recipient));
-
- Map<String, Integer> replies = new HashMap<>();
- replies.put(recipient, ErrorCode.NONE);
- assertMergeOk(replies, Arrays.asList(recipient));
-
- replies.put(recipient, ErrorCode.TRANSIENT_ERROR);
- assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR));
- }
-
- /**
- * Ensures that the current setup will choose the two given recipients, and that it merges similarly to how the
- * {@link DocumentProtocol} would merge these.
- *
- * @param recipientOne The first expected recipient.
- * @param recipientTwo The second expected recipient.
- */
- public void assertMergeTwoReplies(String recipientOne, String recipientTwo) {
- assertSelect(Arrays.asList(recipientOne, recipientTwo));
-
- Map<String, Integer> replies = new HashMap<>();
- replies.put(recipientOne, ErrorCode.NONE);
- replies.put(recipientTwo, ErrorCode.NONE);
- assertMergeOk(replies, Arrays.asList(recipientOne, recipientTwo));
-
- replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR);
- replies.put(recipientTwo, ErrorCode.NONE);
- assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR));
-
- replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR);
- replies.put(recipientTwo, ErrorCode.TRANSIENT_ERROR);
- assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR, ErrorCode.TRANSIENT_ERROR));
-
- replies.put(recipientOne, ErrorCode.NONE);
- replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED);
- assertMergeOk(replies, Arrays.asList(recipientOne));
-
- replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED);
- replies.put(recipientTwo, ErrorCode.NONE);
- assertMergeOk(replies, Arrays.asList(recipientTwo));
-
- replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED);
- replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED);
- assertMergeError(replies, Arrays.asList(DocumentProtocol.ERROR_MESSAGE_IGNORED,
- DocumentProtocol.ERROR_MESSAGE_IGNORED));
- }
-
- /**
- * Waits for a given service pattern to resolve to the given number of hits in the local slobrok.
- *
- * @param pattern The pattern to lookup.
- * @param cnt The number of entries to wait for.
- * @return True if the expected number of entries was found.
- */
- public boolean waitSlobrok(String pattern, int cnt) {
- for (int i = 0; i < 1000 && !Thread.currentThread().isInterrupted(); ++i) {
- Mirror.Entry[] res = net.getMirror().lookup(pattern);
- if (res.length == cnt) {
- return true;
- }
- try { Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ }
- }
- return false;
- }
-
- /**
- * Returns the identity of this frame.
- *
- * @return The ident string.
- */
- public String getIdentity() {
- return identity;
- }
-
- /**
- * Returns the private slobrok server.
- *
- * @return The slobrok.
- */
- public Slobrok getSlobrok() {
- return slobrok;
- }
-
- /**
- * Returns the private message bus.
- *
- * @return The bus.
- */
- public MessageBus getMessageBus() {
- return mbus;
- }
-
- /**
- * Returns the private network layer.
- *
- * @return The network.
- */
- public Network getNetwork() {
- return net;
- }
-
- /**
- * Returns the message being tested.
- *
- * @return The message.
- */
- public Message getMessage() {
- return msg;
- }
-
- /**
- * Sets the message being tested.
- *
- * @param msg The message to set.
- */
- public void setMessage(Message msg) {
- this.msg = msg;
- }
-
- /**
- * Sets the spec of the hop to test with.
- *
- * @param hop The spec to set.
- */
- public void setHop(HopSpec hop) {
- this.hop = hop;
- mbus.setupRouting(new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME).addHop(hop)));
- }
-
- /**
- * Returns the reply receptor used by this frame. All messages tested are tagged with this receptor, so after a
- * successful select, the receptor should contain a non-null reply.
- *
- * @return The reply receptor.
- */
- public Receptor getReceptor() {
- return handler;
- }
-
- /**
- * Implements a dummy network.
- */
- private class MyNetwork extends RPCNetwork {
-
- private List<RoutingNode> nodes = new ArrayList<>();
-
- public MyNetwork(RPCNetworkParams params) {
- super(params);
- }
-
- @Override
- public boolean allocServiceAddress(RoutingNode recipient) {
- recipient.setServiceAddress(new ServiceAddress() { });
- return true;
- }
-
- @Override
- public void freeServiceAddress(RoutingNode recipient) {
- recipient.setServiceAddress(null);
- }
-
- @Override
- public void send(Message msg, List<RoutingNode> recipients) {
- this.nodes.addAll(recipients);
- }
-
- public List<RoutingNode> removeNodes() {
- List<RoutingNode> ret = nodes;
- nodes = new ArrayList<>();
- return ret;
- }
- }
-}
+package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.jrt.slobrok.server.Slobrok; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.network.Identity; +import com.yahoo.messagebus.network.Network; +import com.yahoo.messagebus.network.ServiceAddress; +import com.yahoo.messagebus.network.rpc.RPCNetwork; +import com.yahoo.messagebus.network.rpc.RPCNetworkParams; +import com.yahoo.messagebus.network.rpc.test.TestServer; +import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.test.Receptor; +import com.yahoo.messagebus.test.SimpleProtocol; +import com.yahoo.messagebus.test.SimpleReply; + +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * This is a utility class to allow easier policy test cases. The most important reason to use this is to make sure that + * each test uses a "clean" mbus and slobrok instance. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class PolicyTestFrame extends junit.framework.Assert { + + private final AtomicBoolean destroyed = new AtomicBoolean(false); + private String identity; + private Slobrok slobrok; + private MessageBus mbus; + private MyNetwork net; + private Message msg = null; + private HopSpec hop = null; + private Receptor handler = new Receptor(); + + /** + * Create an anonymous test frame. + * + * @param documentMgr The document manager to use. + */ + public PolicyTestFrame(DocumentTypeManager documentMgr) { + this("anonymous", documentMgr); + } + + /** + * Create a named test frame. + * + * @param identity The identity to use for the server. + * @param documentMgr The document manager to use. + */ + public PolicyTestFrame(String identity, DocumentTypeManager documentMgr) { + this.identity = identity; + try { + slobrok = new Slobrok(); + } catch (ListenFailedException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + net = new MyNetwork(new RPCNetworkParams() + .setIdentity(new Identity(identity)) + .setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); + mbus = new MessageBus(net, new MessageBusParams() + .addProtocol(new DocumentProtocol(documentMgr))); + } + + /** + * Create a test frame running on the same slobrok and mbus as another. + * + * @param frame The frame whose internals to share. + */ + public PolicyTestFrame(PolicyTestFrame frame) { + identity = frame.identity; + slobrok = frame.slobrok; + net = frame.net; + mbus = frame.mbus; + } + + // Inherit doc from Object. + @Override + public void finalize() throws Throwable { + destroy(); + super.finalize(); + } + + /** + * Sets the destroyed flag to true. The very first time this method is called, it cleans up all its dependencies. + * Even if you retain a reference to this object, all of its content is allowed to be garbage collected. + */ + public void destroy() { + if (!destroyed.getAndSet(true)) { + mbus.destroy(); + mbus = null; + net = null; + + slobrok.stop(); + slobrok = null; + } + } + + /** + * Routes the contained message based on the current setup, and returns the leaf send contexts. + * + * @param numExpected The expected number of leaf nodes. + * @return The list of selected send contexts. + */ + public List<RoutingNode> select(int numExpected) { + msg.setRoute(Route.parse(hop.getName())); + new RoutingNode(mbus, net, null, handler, msg).send(); + List<RoutingNode> ret = net.removeNodes(); + assertEquals(numExpected, ret.size()); + return ret; + } + + /** + * Ensures that the current setup selects a given set of routes. + * + * @param expected A list of expected route leaf nodes. + */ + public void assertSelect(List<String> expected) { + if (expected == null) { + assertEquals(0, select(0).size()); + } else { + List<RoutingNode> selected = select(expected.size()); + for (RoutingNode node : selected) { + assertTrue("Route '" + node.getRoute() + "' not selected.", + expected.contains(node.getRoute().toString())); + node.handleReply(new EmptyReply()); + } + } + assertNotNull(handler.getReply(60)); + } + + /** + * This is a convenience method for invoking {@link #assertMerge(Map,List,List)} with no expected value. + * + * @param replies The errors to set in the leaf node replies. + * @param expectedErrors The list of expected errors in the merged reply. + */ + public void assertMergeError(Map<String, Integer> replies, List<Integer> expectedErrors) { + assertMerge(replies, expectedErrors, null); + } + + /** + * This is a convenience method for invoking {@link this#assertMerge(Map,List,List)} with no expected errors. + * + * @param replies The errors to set in the leaf node replies. + * @param allowedValues The list of allowed values in the final reply. + */ + public void assertMergeOk(Map<String, Integer> replies, List<String> allowedValues) { + assertMerge(replies, null, allowedValues); + } + + /** + * Ensures that the current setup generates as many leaf nodes as there are members of the errors argument. Each + * error is then given one of these errors, and the method then ensures that the single returned reply contains the + * given list of expected errors. Finally, if the expected value argument is non-null, this method ensures that the + * reply is a SimpleReply whose string value exists in the allowed list. + * + * @param replies The errors to set in the leaf node replies. + * @param expectedErrors The list of expected errors in the merged reply. + * @param allowedValues The list of allowed values in the final reply. + */ + public void assertMerge(Map<String, Integer> replies, List<Integer> expectedErrors, List<String> allowedValues) { + List<RoutingNode> selected = select(replies.size()); + + for (RoutingNode node : selected) { + String route = node.getRoute().toString(); + assertTrue(replies.containsKey(route)); + Reply ret = new SimpleReply(route); + if (replies.get(route) != ErrorCode.NONE) { + ret.addError(new com.yahoo.messagebus.Error(replies.get(route), route)); + } + node.handleReply(ret); + } + + Reply reply = handler.getReply(60); + assertNotNull(reply); + if (expectedErrors != null) { + assertEquals(expectedErrors.size(), reply.getNumErrors()); + for (int i = 0; i < expectedErrors.size(); ++i) { + assertTrue(expectedErrors.contains(reply.getError(i).getCode())); + } + } else if (reply.hasErrors()) { + StringBuilder err = new StringBuilder("Got unexpected error(s):\n"); + for (int i = 0; i < reply.getNumErrors(); ++i) { + err.append("\t").append(reply.getError(i).toString()); + if (i < reply.getNumErrors() - 1) { + err.append("\n"); + } + } + fail(err.toString()); + } + if (allowedValues != null) { + assertEquals(SimpleProtocol.REPLY, reply.getType()); + assertTrue(allowedValues.contains(((SimpleReply)reply).getValue())); + } else { + assertEquals(0, reply.getType()); + } + } + + /** + * Ensures that the current setup chooses a single recipient, and that it merges similarly to how the + * {@link DocumentProtocol} would merge these. + * + * @param recipient The expected recipient. + */ + public void assertMergeOneReply(String recipient) { + assertSelect(Arrays.asList(recipient)); + + Map<String, Integer> replies = new HashMap<>(); + replies.put(recipient, ErrorCode.NONE); + assertMergeOk(replies, Arrays.asList(recipient)); + + replies.put(recipient, ErrorCode.TRANSIENT_ERROR); + assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR)); + } + + /** + * Ensures that the current setup will choose the two given recipients, and that it merges similarly to how the + * {@link DocumentProtocol} would merge these. + * + * @param recipientOne The first expected recipient. + * @param recipientTwo The second expected recipient. + */ + public void assertMergeTwoReplies(String recipientOne, String recipientTwo) { + assertSelect(Arrays.asList(recipientOne, recipientTwo)); + + Map<String, Integer> replies = new HashMap<>(); + replies.put(recipientOne, ErrorCode.NONE); + replies.put(recipientTwo, ErrorCode.NONE); + assertMergeOk(replies, Arrays.asList(recipientOne, recipientTwo)); + + replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR); + replies.put(recipientTwo, ErrorCode.NONE); + assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR)); + + replies.put(recipientOne, ErrorCode.TRANSIENT_ERROR); + replies.put(recipientTwo, ErrorCode.TRANSIENT_ERROR); + assertMergeError(replies, Arrays.asList(ErrorCode.TRANSIENT_ERROR, ErrorCode.TRANSIENT_ERROR)); + + replies.put(recipientOne, ErrorCode.NONE); + replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED); + assertMergeOk(replies, Arrays.asList(recipientOne)); + + replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED); + replies.put(recipientTwo, ErrorCode.NONE); + assertMergeOk(replies, Arrays.asList(recipientTwo)); + + replies.put(recipientOne, DocumentProtocol.ERROR_MESSAGE_IGNORED); + replies.put(recipientTwo, DocumentProtocol.ERROR_MESSAGE_IGNORED); + assertMergeError(replies, Arrays.asList(DocumentProtocol.ERROR_MESSAGE_IGNORED, + DocumentProtocol.ERROR_MESSAGE_IGNORED)); + } + + /** + * Waits for a given service pattern to resolve to the given number of hits in the local slobrok. + * + * @param pattern The pattern to lookup. + * @param cnt The number of entries to wait for. + * @return True if the expected number of entries was found. + */ + public boolean waitSlobrok(String pattern, int cnt) { + for (int i = 0; i < 1000 && !Thread.currentThread().isInterrupted(); ++i) { + Mirror.Entry[] res = net.getMirror().lookup(pattern); + if (res.length == cnt) { + return true; + } + try { Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ } + } + return false; + } + + /** + * Returns the identity of this frame. + * + * @return The ident string. + */ + public String getIdentity() { + return identity; + } + + /** + * Returns the private slobrok server. + * + * @return The slobrok. + */ + public Slobrok getSlobrok() { + return slobrok; + } + + /** + * Returns the private message bus. + * + * @return The bus. + */ + public MessageBus getMessageBus() { + return mbus; + } + + /** + * Returns the private network layer. + * + * @return The network. + */ + public Network getNetwork() { + return net; + } + + /** + * Returns the message being tested. + * + * @return The message. + */ + public Message getMessage() { + return msg; + } + + /** + * Sets the message being tested. + * + * @param msg The message to set. + */ + public void setMessage(Message msg) { + this.msg = msg; + } + + /** + * Sets the spec of the hop to test with. + * + * @param hop The spec to set. + */ + public void setHop(HopSpec hop) { + this.hop = hop; + mbus.setupRouting(new RoutingSpec().addTable(new RoutingTableSpec(DocumentProtocol.NAME).addHop(hop))); + } + + /** + * Returns the reply receptor used by this frame. All messages tested are tagged with this receptor, so after a + * successful select, the receptor should contain a non-null reply. + * + * @return The reply receptor. + */ + public Receptor getReceptor() { + return handler; + } + + /** + * Implements a dummy network. + */ + private class MyNetwork extends RPCNetwork { + + private List<RoutingNode> nodes = new ArrayList<>(); + + public MyNetwork(RPCNetworkParams params) { + super(params); + } + + @Override + public boolean allocServiceAddress(RoutingNode recipient) { + recipient.setServiceAddress(new ServiceAddress() { }); + return true; + } + + @Override + public void freeServiceAddress(RoutingNode recipient) { + recipient.setServiceAddress(null); + } + + @Override + public void send(Message msg, List<RoutingNode> recipients) { + this.nodes.addAll(recipients); + } + + public List<RoutingNode> removeNodes() { + List<RoutingNode> ret = nodes; + nodes = new ArrayList<>(); + return ret; + } + } +} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java index 8ba217ce15d..c4650dd6099 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java @@ -1,188 +1,188 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test;
-
-import com.yahoo.component.VersionSpecification;
-import com.yahoo.document.DocumentTypeManager;
-import com.yahoo.document.serialization.DocumentDeserializer;
-import com.yahoo.document.serialization.DocumentSerializer;
-import com.yahoo.documentapi.messagebus.protocol.DocumentMessage;
-import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol;
-import com.yahoo.documentapi.messagebus.protocol.DocumentReply;
-import com.yahoo.documentapi.messagebus.protocol.RoutableFactories50;
-import com.yahoo.jrt.ListenFailedException;
-import com.yahoo.jrt.slobrok.server.Slobrok;
-import com.yahoo.messagebus.*;
-import com.yahoo.messagebus.network.Identity;
-import com.yahoo.messagebus.network.rpc.RPCNetworkParams;
-import com.yahoo.messagebus.network.rpc.test.TestServer;
-import com.yahoo.messagebus.routing.Route;
-import com.yahoo.messagebus.test.Receptor;
-import junit.framework.TestCase;
-
-/**
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
- */
-public class RoutableFactoryTestCase extends TestCase {
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Setup
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- private Slobrok slobrok;
- private DocumentProtocol srcProtocol, dstProtocol;
- private TestServer srcServer, dstServer;
- private SourceSession srcSession;
- private DestinationSession dstSession;
-
- @Override
- public void setUp() throws ListenFailedException {
- slobrok = new Slobrok();
- DocumentTypeManager docMan = new DocumentTypeManager();
- dstProtocol = new DocumentProtocol(docMan);
- dstServer = new TestServer(new MessageBusParams().addProtocol(dstProtocol),
- new RPCNetworkParams().setIdentity(new Identity("dst")).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
- dstSession = dstServer.mb.createDestinationSession(new DestinationSessionParams().setName("session").setMessageHandler(new Receptor()));
- srcProtocol = new DocumentProtocol(docMan);
- srcServer = new TestServer(new MessageBusParams().addProtocol(srcProtocol),
- new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok)));
- srcSession = srcServer.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor()));
- assertTrue(srcServer.waitSlobrok("dst/session", 1));
- }
-
- @Override
- public void tearDown() {
- slobrok.stop();
- dstSession.destroy();
- dstServer.destroy();
- srcSession.destroy();
- srcServer.destroy();
- }
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Tests
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- public void testFactory() {
- Route route = Route.parse("dst/session");
-
- // Source should fail to encode the message.
- assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
- Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60);
- assertNotNull(reply);
- System.out.println(reply.getTrace());
- assertTrue(reply.hasErrors());
- assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode());
- assertNull(reply.getError(0).getService());
-
- // Destination should fail to decode the message.
- srcProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification());
- assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
- assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
- System.out.println(reply.getTrace());
- assertTrue(reply.hasErrors());
- assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode());
- assertEquals("dst/session", reply.getError(0).getService());
-
- // Destination should fail to encode the reply.
- dstProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification());
- assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
- Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60);
- assertNotNull(msg);
- reply = new MyReply();
- reply.swapState(msg);
- dstSession.reply(reply);
- assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
- System.out.println(reply.getTrace());
- assertTrue(reply.hasErrors());
- assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode());
- assertEquals("dst/session", reply.getError(0).getService());
-
- // Source should fail to decode the reply.
- dstProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification());
- assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
- assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60));
- reply = new MyReply();
- reply.swapState(msg);
- dstSession.reply(reply);
- assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
- System.out.println(reply.getTrace());
- assertTrue(reply.hasErrors());
- assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode());
- assertNull(reply.getError(0).getService());
-
- // All should succeed.
- srcProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification());
- assertTrue(srcSession.send(new MyMessage(), route).isAccepted());
- assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60));
- reply = new MyReply();
- reply.swapState(msg);
- dstSession.reply(reply);
- assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60));
- System.out.println(reply.getTrace());
- assertFalse(reply.hasErrors());
- }
-
- ////////////////////////////////////////////////////////////////////////////////
- //
- // Utilities
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- private static class MyMessageFactory extends RoutableFactories50.DocumentMessageFactory {
-
- @Override
- protected DocumentMessage doDecode(DocumentDeserializer buf) {
- return new MyMessage();
- }
-
- @Override
- protected boolean doEncode(DocumentMessage msg, DocumentSerializer buf) {
- return true;
- }
- }
-
- private static class MyReplyFactory extends RoutableFactories50.DocumentReplyFactory {
-
- @Override
- protected DocumentReply doDecode(DocumentDeserializer buf) {
- return new MyReply();
- }
-
- @Override
- protected boolean doEncode(DocumentReply msg, DocumentSerializer buf) {
- return true;
- }
- }
-
- private static class MyMessage extends DocumentMessage {
-
- final static int TYPE = 666;
-
- MyMessage() {
- getTrace().setLevel(9);
- }
-
- @Override
- public DocumentReply createReply() {
- return new MyReply();
- }
-
- @Override
- public int getType() {
- return TYPE;
- }
- }
-
- private static class MyReply extends DocumentReply {
-
- final static int TYPE = 777;
-
- public MyReply() {
- super(TYPE);
- }
- }
-}
+package com.yahoo.documentapi.messagebus.protocol.test; + +import com.yahoo.component.VersionSpecification; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentReply; +import com.yahoo.documentapi.messagebus.protocol.RoutableFactories50; +import com.yahoo.jrt.ListenFailedException; +import com.yahoo.jrt.slobrok.server.Slobrok; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.network.Identity; +import com.yahoo.messagebus.network.rpc.RPCNetworkParams; +import com.yahoo.messagebus.network.rpc.test.TestServer; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.test.Receptor; +import junit.framework.TestCase; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RoutableFactoryTestCase extends TestCase { + + //////////////////////////////////////////////////////////////////////////////// + // + // Setup + // + //////////////////////////////////////////////////////////////////////////////// + + private Slobrok slobrok; + private DocumentProtocol srcProtocol, dstProtocol; + private TestServer srcServer, dstServer; + private SourceSession srcSession; + private DestinationSession dstSession; + + @Override + public void setUp() throws ListenFailedException { + slobrok = new Slobrok(); + DocumentTypeManager docMan = new DocumentTypeManager(); + dstProtocol = new DocumentProtocol(docMan); + dstServer = new TestServer(new MessageBusParams().addProtocol(dstProtocol), + new RPCNetworkParams().setIdentity(new Identity("dst")).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); + dstSession = dstServer.mb.createDestinationSession(new DestinationSessionParams().setName("session").setMessageHandler(new Receptor())); + srcProtocol = new DocumentProtocol(docMan); + srcServer = new TestServer(new MessageBusParams().addProtocol(srcProtocol), + new RPCNetworkParams().setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); + srcSession = srcServer.mb.createSourceSession(new SourceSessionParams().setReplyHandler(new Receptor())); + assertTrue(srcServer.waitSlobrok("dst/session", 1)); + } + + @Override + public void tearDown() { + slobrok.stop(); + dstSession.destroy(); + dstServer.destroy(); + srcSession.destroy(); + srcServer.destroy(); + } + + //////////////////////////////////////////////////////////////////////////////// + // + // Tests + // + //////////////////////////////////////////////////////////////////////////////// + + public void testFactory() { + Route route = Route.parse("dst/session"); + + // Source should fail to encode the message. + assertTrue(srcSession.send(new MyMessage(), route).isAccepted()); + Reply reply = ((Receptor)srcSession.getReplyHandler()).getReply(60); + assertNotNull(reply); + System.out.println(reply.getTrace()); + assertTrue(reply.hasErrors()); + assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode()); + assertNull(reply.getError(0).getService()); + + // Destination should fail to decode the message. + srcProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification()); + assertTrue(srcSession.send(new MyMessage(), route).isAccepted()); + assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60)); + System.out.println(reply.getTrace()); + assertTrue(reply.hasErrors()); + assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode()); + assertEquals("dst/session", reply.getError(0).getService()); + + // Destination should fail to encode the reply. + dstProtocol.putRoutableFactory(MyMessage.TYPE, new MyMessageFactory(), new VersionSpecification()); + assertTrue(srcSession.send(new MyMessage(), route).isAccepted()); + Message msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60); + assertNotNull(msg); + reply = new MyReply(); + reply.swapState(msg); + dstSession.reply(reply); + assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60)); + System.out.println(reply.getTrace()); + assertTrue(reply.hasErrors()); + assertEquals(ErrorCode.ENCODE_ERROR, reply.getError(0).getCode()); + assertEquals("dst/session", reply.getError(0).getService()); + + // Source should fail to decode the reply. + dstProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification()); + assertTrue(srcSession.send(new MyMessage(), route).isAccepted()); + assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60)); + reply = new MyReply(); + reply.swapState(msg); + dstSession.reply(reply); + assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60)); + System.out.println(reply.getTrace()); + assertTrue(reply.hasErrors()); + assertEquals(ErrorCode.DECODE_ERROR, reply.getError(0).getCode()); + assertNull(reply.getError(0).getService()); + + // All should succeed. + srcProtocol.putRoutableFactory(MyReply.TYPE, new MyReplyFactory(), new VersionSpecification()); + assertTrue(srcSession.send(new MyMessage(), route).isAccepted()); + assertNotNull(msg = ((Receptor)dstSession.getMessageHandler()).getMessage(60)); + reply = new MyReply(); + reply.swapState(msg); + dstSession.reply(reply); + assertNotNull(reply = ((Receptor)srcSession.getReplyHandler()).getReply(60)); + System.out.println(reply.getTrace()); + assertFalse(reply.hasErrors()); + } + + //////////////////////////////////////////////////////////////////////////////// + // + // Utilities + // + //////////////////////////////////////////////////////////////////////////////// + + private static class MyMessageFactory extends RoutableFactories50.DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + return new MyMessage(); + } + + @Override + protected boolean doEncode(DocumentMessage msg, DocumentSerializer buf) { + return true; + } + } + + private static class MyReplyFactory extends RoutableFactories50.DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new MyReply(); + } + + @Override + protected boolean doEncode(DocumentReply msg, DocumentSerializer buf) { + return true; + } + } + + private static class MyMessage extends DocumentMessage { + + final static int TYPE = 666; + + MyMessage() { + getTrace().setLevel(9); + } + + @Override + public DocumentReply createReply() { + return new MyReply(); + } + + @Override + public int getType() { + return TYPE; + } + } + + private static class MyReply extends DocumentReply { + + final static int TYPE = 777; + + public MyReply() { + super(TYPE); + } + } +} |