diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /dummy-persistence/src |
Publish
Diffstat (limited to 'dummy-persistence/src')
7 files changed, 456 insertions, 0 deletions
diff --git a/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/BucketContents.java b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/BucketContents.java new file mode 100644 index 00000000000..4c76cc908b4 --- /dev/null +++ b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/BucketContents.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.persistence.dummy; + +import com.yahoo.collections.Pair; +import com.yahoo.document.BucketId; +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.persistence.spi.BucketInfo; +import com.yahoo.persistence.spi.DocEntry; +import com.yahoo.persistence.spi.result.GetResult; + +import java.util.ArrayList; +import java.util.List; + +/** + * Class used by DummyPersistence to store its contents. + */ +public class BucketContents { + List<DocEntry> entries = new ArrayList<DocEntry>(); + + BucketInfo.ActiveState active; + + public void setActiveState(BucketInfo.ActiveState state) { + active = state; + } + + public boolean isActive() { + return active == BucketInfo.ActiveState.ACTIVE; + } + + public BucketInfo getBucketInfo() { + int count = 0; + int meta = 0; + int checksum = 0; + + for (DocEntry e : entries) { + if (e.getType() == DocEntry.Type.PUT_ENTRY) { + ++count; + checksum ^= e.getTimestamp(); + } + ++meta; + } + + + return new BucketInfo(checksum, + count, + meta, + meta, + meta, + BucketInfo.ReadyState.READY, + active); + } + + public void put(long timestamp, Document doc) { + for (DocEntry e : entries) { + if (e.getDocumentId().equals(doc.getId())) { + if (e.getTimestamp() > timestamp) { + return; + } + + entries.remove(e); + break; + } + } + + entries.add(new DocEntry(timestamp, doc)); + } + + public boolean remove(long timestamp, DocumentId docId) { + DocEntry found = null; + + for (DocEntry e : entries) { + if ( + e.getType() == DocEntry.Type.PUT_ENTRY && + e.getDocumentId().equals(docId) && + e.getTimestamp() <= timestamp) + { + found = e; + entries.remove(e); + break; + } + } + + entries.add(new DocEntry(timestamp, docId)); + return found != null; + } + + public GetResult get(DocumentId id) { + for (DocEntry e : entries) { + if (e.getType() == DocEntry.Type.PUT_ENTRY && e.getDocumentId().equals(id)) { + return new GetResult(e.getDocument(), e.getTimestamp()); + } + } + + return new GetResult(); + } + + public Pair<BucketContents, BucketContents> split(BucketId target1, BucketId target2) { + BucketContents a = new BucketContents(); + BucketContents b = new BucketContents(); + + for (DocEntry e : entries) { + BucketId bucketId = new BucketIdFactory().getBucketId(e.getDocumentId()); + if (target1.contains(bucketId)) { + a.entries.add(e); + } else { + b.entries.add(e); + } + } + + return new Pair<BucketContents, BucketContents>(a, b); + } + + public BucketContents() {} + + public BucketContents(BucketContents a, BucketContents b) { + if (a != null) { + entries.addAll(a.entries); + } + if (b != null) { + entries.addAll(b.entries); + } + } + +} diff --git a/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/DummyPersistenceProvider.java b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/DummyPersistenceProvider.java new file mode 100644 index 00000000000..3650f8dc5b0 --- /dev/null +++ b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/DummyPersistenceProvider.java @@ -0,0 +1,223 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.persistence.dummy; + +import com.yahoo.collections.Pair; +import com.yahoo.document.fieldset.FieldSet; +import com.yahoo.persistence.spi.AbstractPersistenceProvider; +import com.yahoo.persistence.spi.*; +import com.yahoo.persistence.spi.conformance.ConformanceTest; +import com.yahoo.persistence.spi.result.*; +import java.util.*; +import com.yahoo.document.*; + +/** + * Simple memory-based implementation of the persistence provider interface. + * Intended as an example for future implementations. + */ +public class DummyPersistenceProvider extends AbstractPersistenceProvider +{ + Map<BucketId, BucketContents> bucketContents = new TreeMap<BucketId, BucketContents>(); + long nextIteratorId = 1; + Map<Long, IteratorContext> iteratorContexts = new TreeMap<Long, IteratorContext>(); + + @Override + public synchronized Result initialize() { + bucketContents.clear(); + iteratorContexts.clear(); + return new Result(); + } + + @Override + public synchronized BucketIdListResult listBuckets(short partition) { + return new BucketIdListResult(new ArrayList<BucketId>(bucketContents.keySet())); + } + + @Override + public synchronized BucketInfoResult getBucketInfo(Bucket bucket) { + BucketContents contents = bucketContents.get(bucket.getBucketId()); + if (contents == null) { + return new BucketInfoResult(new BucketInfo()); + } + return new BucketInfoResult(contents.getBucketInfo()); + } + + @Override + public synchronized Result put(Bucket bucket, long timestamp, Document doc) { + bucketContents.get(bucket.getBucketId()).put(timestamp, doc); + return new Result(); + } + + @Override + public synchronized RemoveResult remove(Bucket bucket, long timestamp, DocumentId id) { + return new RemoveResult(bucketContents.get(bucket.getBucketId()).remove(timestamp, id)); + } + + @Override + public synchronized GetResult get(Bucket bucket, FieldSet fieldSet, DocumentId id) { + BucketContents contents = bucketContents.get(bucket.getBucketId()); + if (contents == null) { + return new GetResult(); + } + + return contents.get(id); + } + + @Override + public synchronized CreateIteratorResult createIterator(Bucket bucket, FieldSet fieldSet, Selection selection, PersistenceProvider.IncludedVersions versions) { + nextIteratorId++; + + List<Long> timestamps = new ArrayList<Long>(); + if (selection.getTimestampSubset() == null) { + for (DocEntry e : bucketContents.get(bucket.getBucketId()).entries) { + timestamps.add(e.getTimestamp()); + } + } else { + timestamps.addAll(selection.getTimestampSubset()); + // Explicitly specifying a timestamp subset implies that any version may + // be returned. This is essential for merging to work correctly. + versions = IncludedVersions.ALL_VERSIONS; + } + + iteratorContexts.put(nextIteratorId - 1, new IteratorContext(bucket, fieldSet, selection, timestamps, versions)); + return new CreateIteratorResult(nextIteratorId - 1); + } + + @Override + public synchronized IterateResult iterate(long iteratorId, long maxByteSize) { + IteratorContext context = iteratorContexts.get(iteratorId); + + if (context == null) { + return new IterateResult(Result.ErrorType.PERMANENT_ERROR, "Iterator id not found"); + } + + ArrayList<DocEntry> entries = new ArrayList<DocEntry>(); + for (DocEntry e : bucketContents.get(context.getBucket().getBucketId()).entries) { + if (maxByteSize < 0) { + return new IterateResult(entries, false); + } + + if (context.getTimestamps().contains(e.getTimestamp())) { + context.getTimestamps().remove(e.getTimestamp()); + } else { + continue; + } + + if (e.getType() == DocEntry.Type.PUT_ENTRY) { + + if (context.getSelection() != null && !context.getSelection().match(e.getDocument(), e.getTimestamp())) { + continue; + } + entries.add(e); + maxByteSize -= e.getDocument().getSerializedSize(); + } else if (context.getIncludedVersions() == PersistenceProvider.IncludedVersions.NEWEST_DOCUMENT_OR_REMOVE + || context.getIncludedVersions() == PersistenceProvider.IncludedVersions.ALL_VERSIONS) + { + + if (context.getSelection() != null && !context.getSelection().match(e.getTimestamp())) { + continue; + } + + entries.add(e); + maxByteSize -= e.getDocumentId().toString().length(); + } + } + + return new IterateResult(entries, true); + } + + @Override + public synchronized Result destroyIterator(long iteratorId) { + iteratorContexts.remove(iteratorId); + return new Result(); + } + + @Override + public synchronized Result createBucket(Bucket bucket) { + bucketContents.put(bucket.getBucketId(), new BucketContents()); + return new Result(); + } + + @Override + public synchronized Result deleteBucket(Bucket bucket) { + bucketContents.remove(bucket.getBucketId()); + return new Result(); + } + + private void mergeExistingBucketContentsIntoNew(BucketContents newC, BucketContents oldC) { + if (oldC == null) { + return; + } + Set<Long> newTimestamps = new HashSet<Long>(); + for (DocEntry entry : newC.entries) { + newTimestamps.add(entry.getTimestamp()); + } + // Don't overwrite new entries with old ones + for (DocEntry oldEntry : oldC.entries) { + if (newTimestamps.contains(oldEntry.getTimestamp())) { + continue; + } + newC.entries.add(oldEntry); + } + } + + @Override + public synchronized Result split(Bucket source, Bucket target1, Bucket target2) { + BucketContents sourceContent = bucketContents.get(source.getBucketId()); + BucketContents existingTarget1 = bucketContents.get(target1.getBucketId()); + BucketContents existingTarget2 = bucketContents.get(target2.getBucketId()); + + Pair<BucketContents, BucketContents> contents + = sourceContent.split(target1.getBucketId(), target2.getBucketId()); + + bucketContents.remove(source.getBucketId()); + mergeExistingBucketContentsIntoNew(contents.getFirst(), existingTarget1); + mergeExistingBucketContentsIntoNew(contents.getSecond(), existingTarget2); + + BucketInfo.ActiveState targetActiveState + = (sourceContent.getBucketInfo().isActive() + ? BucketInfo.ActiveState.ACTIVE + : BucketInfo.ActiveState.NOT_ACTIVE); + contents.getFirst().setActiveState(targetActiveState); + contents.getSecond().setActiveState(targetActiveState); + + bucketContents.put(target1.getBucketId(), contents.getFirst()); + bucketContents.put(target2.getBucketId(), contents.getSecond()); + + return new Result(); + } + + @Override + public synchronized Result join(Bucket source1, Bucket source2, Bucket target) { + BucketInfo.ActiveState activeState = BucketInfo.ActiveState.NOT_ACTIVE; + BucketContents targetExisting = bucketContents.get(target.getBucketId()); + BucketContents sourceExisting1 = bucketContents.get(source1.getBucketId()); + BucketContents sourceExisting2 = null; + boolean singleBucketJoin = source2.getBucketId().equals(source1.getBucketId()); + if (!singleBucketJoin) { + sourceExisting2 = bucketContents.get(source2.getBucketId()); + } + + if (sourceExisting1 != null && sourceExisting1.isActive()) { + activeState = BucketInfo.ActiveState.ACTIVE; + } + if (sourceExisting2 != null && sourceExisting2.isActive()) { + activeState = BucketInfo.ActiveState.ACTIVE; + } + + BucketContents contents = new BucketContents(sourceExisting1, sourceExisting2); + bucketContents.remove(source1.getBucketId()); + if (sourceExisting2 != null) { + bucketContents.remove(source2.getBucketId()); + } + mergeExistingBucketContentsIntoNew(contents, targetExisting); + contents.setActiveState(activeState); + bucketContents.put(target.getBucketId(), contents); + return new Result(); + } + + @Override + public synchronized Result setActiveState(Bucket bucket, BucketInfo.ActiveState active) { + bucketContents.get(bucket.getBucketId()).setActiveState(active); + return new Result(); + } +} diff --git a/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/DummyPersistenceProviderHandler.java b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/DummyPersistenceProviderHandler.java new file mode 100644 index 00000000000..ea36f158f62 --- /dev/null +++ b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/DummyPersistenceProviderHandler.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.persistence.dummy; + +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.persistence.rpc.PersistenceProviderHandler; + +public class DummyPersistenceProviderHandler { + DummyPersistenceProvider provider; + + public DummyPersistenceProviderHandler(PersistenceProviderHandler rpcHandler, DocumentmanagerConfig docManConfig) { + provider = new DummyPersistenceProvider(); + rpcHandler.initialize(provider, new DocumentTypeManager(docManConfig)); + } +} diff --git a/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/IteratorContext.java b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/IteratorContext.java new file mode 100644 index 00000000000..ac39fd8f670 --- /dev/null +++ b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/IteratorContext.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.persistence.dummy; + +import com.yahoo.document.fieldset.FieldSet; +import com.yahoo.persistence.spi.Bucket; +import com.yahoo.persistence.spi.PersistenceProvider; +import com.yahoo.persistence.spi.Selection; + +import java.util.List; + +/** + * Class to represent an ongoing iterator in dummy persistence. + */ +public class IteratorContext { + List<Long> timestamps; + + public FieldSet getFieldSet() { + return fieldSet; + } + + private FieldSet fieldSet; + + public Bucket getBucket() { + return bucket; + } + + private Bucket bucket; + + public Selection getSelection() { + return selection; + } + + private Selection selection; + + public PersistenceProvider.IncludedVersions getIncludedVersions() { + return includedVersions; + } + + private PersistenceProvider.IncludedVersions includedVersions; + + IteratorContext(Bucket bucket, FieldSet fieldSet, Selection selection, + List<Long> timestamps, + PersistenceProvider.IncludedVersions versions) { + this.fieldSet = fieldSet; + this.bucket = bucket; + this.selection = selection; + this.includedVersions = versions; + this.timestamps = timestamps; + } + + public List<Long> getTimestamps() { return timestamps; } +} diff --git a/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/package-info.java b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/package-info.java new file mode 100644 index 00000000000..4155537e41b --- /dev/null +++ b/dummy-persistence/src/main/java/com/yahoo/persistence/dummy/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.persistence.dummy; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/dummy-persistence/src/test/config/.gitignore b/dummy-persistence/src/test/config/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/dummy-persistence/src/test/config/.gitignore diff --git a/dummy-persistence/src/test/java/com/yahoo/persistence/dummy/DummyPersistenceTest.java b/dummy-persistence/src/test/java/com/yahoo/persistence/dummy/DummyPersistenceTest.java new file mode 100644 index 00000000000..6fa3af9b40d --- /dev/null +++ b/dummy-persistence/src/test/java/com/yahoo/persistence/dummy/DummyPersistenceTest.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.persistence.dummy; + +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.persistence.PersistenceRpcConfig; +import com.yahoo.persistence.rpc.PersistenceProviderHandler; +import com.yahoo.persistence.spi.PersistenceProvider; +import com.yahoo.persistence.spi.conformance.ConformanceTest; + +public class DummyPersistenceTest extends ConformanceTest { + + class DummyPersistenceFactory implements PersistenceProviderFactory { + + @Override + public PersistenceProvider createProvider(DocumentTypeManager manager) { + return new DummyPersistenceProvider(); + } + + @Override + public boolean supportsActiveState() { + return true; + } + } + + public void testConstruct() { + DummyPersistenceProviderHandler provider = new DummyPersistenceProviderHandler( + new PersistenceProviderHandler(new PersistenceRpcConfig(new PersistenceRpcConfig.Builder())), null); + } + + public void testConformance() throws Exception { + doConformanceTest(new DummyPersistenceFactory()); + } +} |