diff options
Diffstat (limited to 'document/src/main/java')
196 files changed, 25289 insertions, 0 deletions
diff --git a/document/src/main/java/com/yahoo/document/ArrayDataType.java b/document/src/main/java/com/yahoo/document/ArrayDataType.java new file mode 100644 index 00000000000..640bd94bd1c --- /dev/null +++ b/document/src/main/java/com/yahoo/document/ArrayDataType.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.Array; +import com.yahoo.vespa.objects.Ids; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ArrayDataType extends CollectionDataType { + // The global class identifier shared with C++. + public static int classId = registerClass(Ids.document + 54, ArrayDataType.class); + + public ArrayDataType(DataType nestedType) { + super("Array<"+nestedType.getName()+">", 0, nestedType); + setId(getName().toLowerCase().hashCode()); + } + + public ArrayDataType(DataType nestedType, int code) { + super("Array<"+nestedType.getName()+">", code, nestedType); + } + + public ArrayDataType clone() { + return (ArrayDataType) super.clone(); + } + + public Array createFieldValue() { + return new Array(this); + } + + @Override + public Class getValueClass() { + return Array.class; + } + + @Override + public FieldPath buildFieldPath(String remainFieldName) + { + if (remainFieldName.length() > 0 && remainFieldName.charAt(0) == '[') { + int endPos = remainFieldName.indexOf(']'); + if (endPos == -1) { + throw new IllegalArgumentException("Array subscript must be closed with ]"); + } else { + FieldPath path = getNestedType().buildFieldPath(skipDotInString(remainFieldName, endPos)); + List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(path.getList()); + if (remainFieldName.charAt(1) == '$') { + tmpPath.add(0, FieldPathEntry.newVariableLookupEntry(remainFieldName.substring(2, endPos), getNestedType())); + } else { + tmpPath.add(0, FieldPathEntry.newArrayLookupEntry(Integer.parseInt(remainFieldName.substring(1, endPos)), getNestedType())); + } + + return new FieldPath(tmpPath); + } + } + + return getNestedType().buildFieldPath(remainFieldName); + } + +} diff --git a/document/src/main/java/com/yahoo/document/BaseStructDataType.java b/document/src/main/java/com/yahoo/document/BaseStructDataType.java new file mode 100755 index 00000000000..76ee98331ab --- /dev/null +++ b/document/src/main/java/com/yahoo/document/BaseStructDataType.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Date: Apr 15, 2008 + * + * @author humbe + */ +public abstract class BaseStructDataType extends StructuredDataType { + + protected Map<Integer, Field> fieldIds = new LinkedHashMap<>(); + protected Map<Integer, Field> fieldIdsV6 = new LinkedHashMap<>(); + protected Map<String, Field> fields = new LinkedHashMap<>(); + + protected Compressor compressor = new Compressor(CompressionType.NONE); + + BaseStructDataType(String name) { + super(name); + } + + BaseStructDataType(int id, String name) { + super(id, name); + } + + protected void assign(BaseStructDataType type) { + BaseStructDataType stype = type.clone(); + + fieldIds = stype.fieldIds; + fieldIdsV6 = stype.fieldIdsV6; + fields = stype.fields; + } + + @Override + public BaseStructDataType clone() { + BaseStructDataType type = (BaseStructDataType) super.clone(); + type.fieldIds = new LinkedHashMap<>(); + type.fieldIdsV6 = new LinkedHashMap<>(); + + type.fields = new LinkedHashMap<>(); + for (Field field : fieldIds.values()) { + type.fields.put(field.getName(), field); + type.fieldIds.put(field.getId(Document.SERIALIZED_VERSION), field); + type.fieldIdsV6.put(field.getId(6), field); + } + return type; + } + + public void addField(Field field) { + if (fields.containsKey(field.getName())) { + throw new IllegalArgumentException("Struct " + getName() + " already contains field with name " + field.getName()); + } + if (fieldIds.containsKey(field.getId(Document.SERIALIZED_VERSION))) { + throw new IllegalArgumentException("Struct " + getName() + " already contains field with id " + field.getId(Document.SERIALIZED_VERSION)); + } + if (fieldIdsV6.containsKey(field.getId(6))) { + throw new IllegalArgumentException("Struct " + getName() + " already contains a field with deprecated document serialization id " + field.getId(6)); + } + + fields.put(field.getName(), field); + fieldIds.put(field.getId(Document.SERIALIZED_VERSION), field); + fieldIdsV6.put(field.getId(6), field); + } + + public Field removeField(String fieldName) { + Field old = fields.remove(fieldName); + if (old != null) { + fieldIds.remove(old.getId(Document.SERIALIZED_VERSION)); + fieldIdsV6.remove(old.getId(6)); + } + return old; + } + + public void clearFields() { + fieldIds.clear(); + fieldIdsV6.clear(); + fields.clear(); + } + + public Field getField(Integer fieldId, int version) { + if (version > 6) { + return fieldIds.get(fieldId); + } else { + return fieldIdsV6.get(fieldId); + } + } + + @Override + public Field getField(String fieldName) { + return fields.get(fieldName); + } + + @Override + public Field getField(int id) { + return fieldIds.get(id); + } + + public boolean hasField(Field field, int version) { + Field f = getField(field.getId(version), version); + return f != null && f.equals(field); + } + + public boolean hasField(String name) { + return fields.containsKey(name); + } + + public boolean hasField(Field f) { + if (hasField(f, 6) || hasField(f, Document.SERIALIZED_VERSION)) { + return true; + } + return false; + } + + @Override + public Collection<Field> getFields() { + return fields.values(); + } + + public int getFieldCount() { + return fields.size(); + } + + /** Returns the compressor to use to compress data of this type */ + public Compressor getCompressor() { return compressor; } + + /** Returns a view of the configuration of the compressor used to compress this type */ + public CompressionConfig getCompressionConfig() { + // CompressionConfig accepts a percentage (but exposes a factor) ... + float compressionThresholdPercentage = (float)compressor.compressionThresholdFactor() * 100; + + return new CompressionConfig(compressor.type(), + compressor.level(), + compressionThresholdPercentage, + compressor.compressMinSizeBytes()); + } + + /** Set the config to the compressor used to compress data of this type */ + public void setCompressionConfig(CompressionConfig config) { + CompressionType type = config.type; + compressor = new Compressor(type, + config.compressionLevel, + config.thresholdFactor(), + (int)config.minsize); + } + +} diff --git a/document/src/main/java/com/yahoo/document/BucketDistribution.java b/document/src/main/java/com/yahoo/document/BucketDistribution.java new file mode 100644 index 00000000000..bb4792b5982 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/BucketDistribution.java @@ -0,0 +1,205 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document;
+
+import com.yahoo.document.BucketId;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class BucketDistribution {
+
+ // A logger object to enable proper logging.
+ private static Logger log = Logger.getLogger(BucketDistribution.class.getName());
+
+ // A map from bucket id to column index.
+ private int[] bucketToColumn;
+
+ // The number of columns to distribute to.
+ private int numColumns;
+
+ // The number of bits to use for bucket identification.
+ private int numBucketBits;
+
+ /**
+ * Constructs a new bucket distribution object with a given number of columns and buckets.
+ *
+ * @param numColumns The number of columns to distribute to.
+ * @param numBucketBits The number of bits to use for bucket id.
+ */
+ public BucketDistribution(int numColumns, int numBucketBits) {
+ this.numBucketBits = numBucketBits;
+ bucketToColumn = new int[getNumBuckets()];
+ reset();
+ setNumColumns(numColumns);
+ }
+
+ /**
+ * Constructs a new bucket distribution object as a copy of another.
+ *
+ * @param other The distribution object to copy.
+ */
+ public BucketDistribution(BucketDistribution other) {
+ bucketToColumn = other.bucketToColumn.clone();
+ numColumns = other.numColumns;
+ numBucketBits = other.numBucketBits;
+ }
+
+ /**
+ * Returns the number of buckets that the given number of bucket bits will allow.
+ *
+ * @param numBucketBits The number of bits to use for bucket id.
+ * @return The number of buckets allowed.
+ */
+ private static int getNumBuckets(int numBucketBits) {
+ return 1 << numBucketBits;
+ }
+
+ /**
+ * This method returns a list that contains the distributions of the given number of buckets over the given number
+ * of columns.
+ *
+ * @param numColumns The number of columns to distribute to.
+ * @param numBucketBits The number of bits to use for bucket id.
+ * @return The bucket distribution.
+ */
+ private static List<Integer> getBucketCount(int numColumns, int numBucketBits) {
+ List<Integer> ret = new ArrayList<Integer>(numColumns);
+ int cnt = getNumBuckets(numBucketBits) / numColumns;
+ int rst = getNumBuckets(numBucketBits) % numColumns;
+ for (int i = 0; i < numColumns; ++i) {
+ ret.add(cnt + (i < rst ? 1 : 0));
+ }
+ return ret;
+ }
+
+ /**
+ * This method returns a list similar to {@link BucketDistribution#getBucketCount(int, int)}, except that the returned list
+ * contains the number of buckets that will have to be migrated from each column if an additional column was added.
+ *
+ * @param numColumns The original number of columns.
+ * @param numBucketBits The number of bits to use for bucket id.
+ * @return The number of buckets to migrate, one value per column.
+ */
+ private static List<Integer> getBucketMigrateCount(int numColumns, int numBucketBits) {
+ List<Integer> ret = getBucketCount(numColumns++, numBucketBits);
+ int cnt = getNumBuckets(numBucketBits) / numColumns;
+ int rst = getNumBuckets(numBucketBits) % numColumns;
+ for (int i = 0; i < numColumns - 1; ++i) {
+ ret.set(i, ret.get(i) - (cnt + (i < rst ? 1 : 0)));
+ }
+ return ret;
+ }
+
+ /**
+ * Sets the number of columns to distribute to to 1, and resets the content of the internal bucket-to-column map so
+ * that it all buckets point to that single column.
+ */
+ public void reset() {
+ for (int i = 0; i < bucketToColumn.length; ++i) {
+ bucketToColumn[i] = 0;
+ }
+ numColumns = 1;
+ }
+
+ /**
+ * Adds a single column to this bucket distribution object. This will modify the internal bucket-to-column map so
+ * that it takes into account the new column.
+ */
+ private void addColumn() {
+ int newColumns = numColumns + 1;
+ List<Integer> migrate = getBucketMigrateCount(numColumns, numBucketBits);
+ int numBuckets = getNumBuckets(numBucketBits);
+ for (int i = 0; i < numBuckets; ++i) {
+ int old = bucketToColumn[i];
+ if (migrate.get(old) > 0) {
+ bucketToColumn[i] = numColumns; // move this bucket to the new column
+ migrate.set(old, migrate.get(old) - 1);
+ }
+ }
+ numColumns = newColumns;
+ }
+
+ /**
+ * Sets the number of columns to use for this document distribution object. This will reset and setup this object
+ * from scratch. The original number of buckets is maintained.
+ *
+ * @param numColumns The new number of columns to distribute to.
+ */
+ public synchronized void setNumColumns(int numColumns) {
+ if (numColumns < this.numColumns) {
+ reset();
+ }
+ if (numColumns == this.numColumns) {
+ return;
+ }
+ for (int i = numColumns - this.numColumns; --i >= 0; ) {
+ addColumn();
+ }
+ }
+
+ /**
+ * Returns the number of columns to distribute to.
+ *
+ * @return The number of columns.
+ */
+ public int getNumColumns() {
+ return numColumns;
+ }
+
+ /**
+ * Sets the number of buckets to use for this document distribution object. This will reset and setup this object
+ * from scratch. The original number of columns is maintained.
+ *
+ * @param numBucketBits The new number of bits to use for bucket id.
+ */
+ public synchronized void setNumBucketBits(int numBucketBits) {
+ if (numBucketBits == this.numBucketBits) {
+ return;
+ }
+ this.numBucketBits = numBucketBits;
+ bucketToColumn = new int[getNumBuckets(numBucketBits)];
+ int numColumns = this.numColumns;
+ reset();
+ setNumColumns(numColumns);
+ }
+
+ /**
+ * Returns the number of bits used for bucket identifiers.
+ *
+ * @return The number of bits.
+ */
+ public int getNumBucketBits() {
+ return numBucketBits;
+ }
+
+ /**
+ * Returns the number of buckets available using the configured number of bucket bits.
+ *
+ * @return The number of buckets.
+ */
+ public int getNumBuckets() {
+ return getNumBuckets(numBucketBits);
+ }
+
+ /**
+ * This method maps the given bucket id to its corresponding column.
+ *
+ * @param bucketId The bucket whose column to lookup.
+ * @return The column to distribute the bucket to.
+ */
+ public int getColumn(BucketId bucketId) {
+ int ret = (int)(bucketId.getId() & (getNumBuckets(numBucketBits) - 1));
+ if (ret >= bucketToColumn.length) {
+ log.log(Level.SEVERE,
+ "The bucket distribution map is not in sync with the number of bucket bits. " +
+ "This should never happen! Distribution is broken!!");
+ return 0;
+ }
+ return bucketToColumn[ret];
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/BucketId.java b/document/src/main/java/com/yahoo/document/BucketId.java new file mode 100755 index 00000000000..d0e360ddb2d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/BucketId.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +/** + * Representation of a bucket identifier. + */ +public class BucketId implements Comparable<BucketId> { + public static final int COUNT_BITS = 6; + private static final long STRIP_MASK = 0xFC000000FFFFFFFFl; + private long id = 0; + private static long[] usedMask; + + static { + usedMask = new long[59]; + long val = 0; + for (int i=0; i<usedMask.length; ++i) { + usedMask[i] = val; + val = (val << 1) | 1; + } + } + + /** + * Default-constructed BucketId signifies an invalid bucket ID. + */ + public BucketId() { + } + + /** + * Creates a bucket id with the given raw bucket id. This is a 64 bit mask + * where the first 6 MSB bits set how many LSB bits should actually be used. + * Right now it only have simple functionality. More will be added for it + * to be configurable. + */ + public BucketId(long id) { + this.id = id; + } + + public BucketId(int usedBits, long id) { + long usedMask = ((long) usedBits) << (64 - COUNT_BITS); + id <<= COUNT_BITS; + id >>>= COUNT_BITS; + this.id = id | usedMask; + } + + public BucketId(String serialized) { + if (!serialized.startsWith("BucketId(0x")) { + throw new IllegalArgumentException("Serialized bucket id must start with 'BucketId(0x'"); + } + if (!serialized.endsWith(")")) { + throw new IllegalArgumentException("Serialized bucket id must end with ')'"); + } + + // Parse hex string after "0x" + int index; + char c; + long id = 0; + for (index = 11; index < serialized.length()-1; index++) { + c = serialized.charAt(index); + if (!((c>=48 && c<=57) || // digit + (c>=97 && c<=102))) { // a-f + throw new IllegalArgumentException("Serialized bucket id (" + serialized + ") contains illegal character at position " + index); + } + id <<= 4; + id += Integer.parseInt(String.valueOf(c),16); + } + this.id = id; + if (getUsedBits() == 0) { + throw new IllegalArgumentException("Created bucket id "+id+", but no countbits are set"); + } + } + + public boolean equals(Object o) { + return (o instanceof BucketId && ((BucketId) o).getId() == this.getId()); + } + + public int compareTo(BucketId other) { + if (id >>> 32 == other.id >>> 32) { + if ((id & 0xFFFFFFFFl) > (other.id & 0xFFFFFFFFl)) { + return 1; + } else if ((id & 0xFFFFFFFFl) < (other.id & 0xFFFFFFFFl)) { + return -1; + } + return 0; + } else if ((id >>> 32) > (other.id >>> 32)) { + return 1; + } else { + return -1; + } + } + + public int hashCode() { + return (int) id; + } + + public int getUsedBits() { return (int) (id >>> (64 - COUNT_BITS)); } + + public long getRawId() { return id; } + + public long getId() { + int notUsed = 64 - getUsedBits(); + long usedMask = (0xFFFFFFFFFFFFFFFFl << notUsed) >>> notUsed; + long countMask = (0xFFFFFFFFFFFFFFFFl >>> (64 - COUNT_BITS)) << (64 - COUNT_BITS); + return id & (usedMask | countMask); + } + + public long withoutCountBits() { + return id & usedMask[getUsedBits()]; + } + + public String toString() { + StringBuilder sb = new StringBuilder().append("BucketId(0x"); + String number = Long.toHexString(getId()); + for (int i=number.length(); i<16; ++i) { + sb.append('0'); + } + sb.append(number).append(')'); + return sb.toString(); + } + + public boolean contains(BucketId id) { + if (id.getUsedBits() < getUsedBits()) { + return false; + } + BucketId copy = new BucketId(getUsedBits(), id.getRawId()); + return (copy.getId() == getId()); + } + + public boolean contains(DocumentId docId, BucketIdFactory factory) { + return contains(factory.getBucketId(docId)); + } +} diff --git a/document/src/main/java/com/yahoo/document/BucketIdFactory.java b/document/src/main/java/com/yahoo/document/BucketIdFactory.java new file mode 100644 index 00000000000..f327d907448 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/BucketIdFactory.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.idstring.*; + +/** + * A bucket id contains bit used for various purposes. In most use cases, these can use the default + * settings, but the number of bits used for the different purposes is configurable, to allow for + * special uses. + * + * Because of this, bucket ids cannot be generated without knowing how the bucket id is configured to + * be put together, so all bucket ids must be generated by this factory class. + * + * For more information about what the sub parts of a bucket id actually is, read the bucket splitting documentation. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class BucketIdFactory { + + private final int gidBits; + private final int locationBits; + private final int countBits; + + private final long initialCount; + private final long locationMask; + private final long gidMask; + + /** Create a factory, using the default configuration. */ + public BucketIdFactory() { + this(32, 26, 6); + } + + /** + * Create a factory, using the provided configuration. + * @param gidBits How many bits that are used to specify gidbits. + */ + public BucketIdFactory(int locationBits, int gidBits, int countBits) { + this.locationBits = locationBits; + this.gidBits = gidBits; + this.countBits = countBits; + initialCount = 58l << (64 - countBits); + locationMask = 0xFFFFFFFFFFFFFFFFl >>> (64 - getLocationBitCount()); + gidMask = ((0xFFFFFFFFFFFFFFFFl >>> getLocationBitCount()) << (getLocationBitCount() + BucketId.COUNT_BITS)) >>> BucketId.COUNT_BITS; + + } + + /** + * Create a factory, with parameters gotten from configuration. + * TODO: Not implemented yet + * @param configId The config id from where to get config. + */ + public BucketIdFactory(String configId) { + this(32, 26, 6); + } + + /** @return Get number of bits used for storing of LSB part of location.*/ + public int getLocationBitCount() { return locationBits; } + + /** @return Get number of bits used to specify gid. */ + public int getGidBitCount() { return gidBits; } + + /** @return Get number of bits used to store bit count used. */ + public int getCountBitCount() { return countBits; } + + /** + * Get the gid bit contribution in the bucket id, shifted to the correct + * position in the id. + * + * @param gid The gid we need to calculate contribution from. + * @return A mask to or with the bucket id to get the bit set. + */ + private long getGidContribution(byte[] gid) { + long gidbits = 0; + for (int i=4; i<12; ++i) { + gidbits <<= 8; + long tall = gid[15 - i] & 0xFFl; + assert(tall >= 0 && tall <= 255); + gidbits |= tall; + } + return gidbits & gidMask; + } + + /** + * Get the bucket id for a given document. + * + * @param doc The doc. + * @return The bucket id. + */ + public BucketId getBucketId(DocumentId doc) { + long location = doc.getScheme().getLocation(); + byte[] gid = doc.getGlobalId(); + + long gidContribution = getGidContribution(gid); + + IdString.GidModifier gm = doc.getScheme().getGidModifier(); + if (gm != null && gm.usedBits != 0) { + gidContribution &= (0xFFFFFFFFFFFFFFFFl << (gm.usedBits + getLocationBitCount())); + gidContribution |= (gm.value << getLocationBitCount()); + } + + return new BucketId(64 - BucketId.COUNT_BITS, initialCount | (gidMask & gidContribution) | (locationMask & location)); + } + +} diff --git a/document/src/main/java/com/yahoo/document/CollectionDataType.java b/document/src/main/java/com/yahoo/document/CollectionDataType.java new file mode 100644 index 00000000000..87f1f2947bf --- /dev/null +++ b/document/src/main/java/com/yahoo/document/CollectionDataType.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.objects.Ids; +import com.yahoo.vespa.objects.ObjectVisitor; + +import java.util.List; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public abstract class CollectionDataType extends DataType { + // The global class identifier shared with C++. + public static int classId = registerClass(Ids.document + 53, CollectionDataType.class); + + private DataType nestedType; + + protected CollectionDataType(String name, int code, DataType nestedType) { + super(name, code); + this.nestedType = nestedType; + } + + @Override + public abstract CollectionFieldValue createFieldValue(); + + @Override + public CollectionDataType clone() { + CollectionDataType type = (CollectionDataType) super.clone(); + type.nestedType = nestedType.clone(); + return type; + } + + @SuppressWarnings("deprecation") + public DataType getNestedType() { + return nestedType; + } + + @Override + protected FieldValue createByReflection(Object arg) { return null; } + + /** + * Sets the nested type of this CollectionDataType. WARNING! Do not use! Only to be used by config system! + */ + public void setNestedType(DataType nestedType) { + this.nestedType = nestedType; + } + + @Override + public PrimitiveDataType getPrimitiveType() { + return nestedType.getPrimitiveType(); + } + + @Override + public boolean isValueCompatible(FieldValue value) { + if (!(value instanceof CollectionFieldValue)) { + return false; + } + CollectionFieldValue cfv = (CollectionFieldValue) value; + if (equals(cfv.getDataType())) { + //the field value if of this type: + return true; + } + return false; + } + + @Override + protected void register(DocumentTypeManager manager, List<DataType> seenTypes) { + seenTypes.add(this); + if (!seenTypes.contains(getNestedType())) { + //we haven't seen this one before, register it: + getNestedType().register(manager, seenTypes); + } + super.register(manager, seenTypes); + } + + @Override + public void visitMembers(ObjectVisitor visitor) { + super.visitMembers(visitor); + visitor.visit("nestedType", nestedType); + } + + @Override + public boolean isMultivalue() { return true; } + +} diff --git a/document/src/main/java/com/yahoo/document/CompressionConfig.java b/document/src/main/java/com/yahoo/document/CompressionConfig.java new file mode 100644 index 00000000000..c827ea23b03 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/CompressionConfig.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.compress.CompressionType; + +import java.io.Serializable; + +public class CompressionConfig implements Serializable { + + public CompressionConfig(CompressionType type, + int level, + float threshold, + long minSize) + { + this.type = type; + this.compressionLevel = level; + this.threshold = threshold; + this.minsize = minSize; + } + + public CompressionConfig() { + this(CompressionType.NONE, 9, 95, 0); + } + + public CompressionConfig(CompressionType type) { + this(type, 9, 95, 0); + } + + public CompressionConfig(CompressionType type, int level, float threshold) { + this(type, level, threshold, 0); + } + + public final CompressionType type; + public int compressionLevel; + public float threshold; + public final long minsize; + + /** get a multiplier for comparing compressed and original size */ + public float thresholdFactor() { return 0.01f * threshold; } +} diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java new file mode 100644 index 00000000000..6e6103c61fd --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -0,0 +1,325 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.collections.Pair; +import com.yahoo.concurrent.CopyOnWriteHashMap; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.UriFieldValue; +import com.yahoo.vespa.objects.Identifiable; +import com.yahoo.vespa.objects.Ids; +import com.yahoo.vespa.objects.ObjectVisitor; + +import java.io.Serializable; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.LinkedList; +import java.util.List; + +/** + * Enumeration of the possible types of fields. Since arrays and weighted sets may be defined for any types, including + * themselves, this enumeration is open ended. + * + * @author bratseth + */ +public abstract class DataType extends Identifiable implements Serializable, Comparable<DataType> { + + // The global class identifier shared with C++. + public static int classId = registerClass(Ids.document + 50, DataType.class); + + // NOTE: These types are also defined in + // document/src/vespa/document/datatype/datatype.h + // Changes here must also be done there + + public final static NumericDataType NONE = new NumericDataType("none", -1, IntegerFieldValue.class, IntegerFieldValue.getFactory()); + public final static NumericDataType INT = new NumericDataType("int", 0, IntegerFieldValue.class, IntegerFieldValue.getFactory()); + public final static NumericDataType FLOAT = new NumericDataType("float", 1, FloatFieldValue.class, FloatFieldValue.getFactory()); + public final static PrimitiveDataType STRING = new PrimitiveDataType("string", 2, StringFieldValue.class, StringFieldValue.getFactory()); + public final static PrimitiveDataType RAW = new PrimitiveDataType("raw", 3, Raw.class, Raw.getFactory()); + public final static NumericDataType LONG = new NumericDataType("long", 4, LongFieldValue.class, LongFieldValue.getFactory()); + public final static NumericDataType DOUBLE = new NumericDataType("double", 5, DoubleFieldValue.class, DoubleFieldValue.getFactory()); + // ARRAY is type 6, but never used, array IDs are generated + // public final static PrimitiveDataType FIELDMAP = new PrimitiveDataType("FieldMap", 7, FieldMap.class); + public final static DocumentType DOCUMENT = new DocumentType("document"); + // Not used anymore : public final static NumericDataType TIMESTAMP = new NumericDataType("Timestamp", 9, LongFieldValue.class); + public final static PrimitiveDataType URI = new PrimitiveDataType("uri", 10, UriFieldValue.class, new UriFieldValue.Factory()); + // Not used anymore : public final static PrimitiveDataType EXACTSTRING = new PrimitiveDataType("ExactString", 11, StringFieldValue.class); + // Not used anymore: public final static PrimitiveDataType CONTENT = new PrimitiveDataType("content", 12, Content.class, new Content.Factory()); + public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory()); + // WEIGHTEDSET is type 17, but never used, weighted set IDs are generated + // Tags are converted to weightedset<string> when reading the search definition + public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true); + // Not yet, just reserve id 19. public final static NumericDataType SHORT = new NumericDataType("Int", 19, ShortFieldValue.class); + // Guess I'll say STRUCT is 19 though, although I never intend to use it for anything as it has to be autogenerated now.. + // Let's say that AnnotationReference is 20, but those types will be generated from AnnotationReferenceDataType + public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory()); + public final static PrimitiveDataType TENSOR = new PrimitiveDataType("tensor", 21, TensorFieldValue.class, TensorFieldValue.getFactory()); + + public static int lastPredefinedDataTypeId() { + return 21; + } + + /** + * Set to true when this type is registered in a type manager. From that time we should refuse changes. + */ + private boolean registered = false; + + private String name; + + /** + * The id of this type + */ + private int dataTypeId; + + static final private CopyOnWriteHashMap<Pair, Constructor> constructorCache = new CopyOnWriteHashMap<>(); + /** + * Creates a datatype + * + * @param name the name of the type + * @param dataTypeId the id of the type + */ + protected DataType(java.lang.String name, int dataTypeId) { + this.name = name; + this.dataTypeId = dataTypeId; + } + + @SuppressWarnings("CloneDoesntDeclareCloneNotSupportedException") + public DataType clone() { + return (DataType)super.clone(); + } + + public void setRegistered() { + registered = true; + } + + public boolean isRegistered() { + return registered; + } + + /** + * Creates a new, empty FieldValue of this type. + * + * @return a new, empty FieldValue of this type. + */ + public abstract FieldValue createFieldValue(); + + /** + * This will try to create the object by reflection. This can be very expensive + * so some might discourage that. + * @param arg The constructor argument. + * @return Fully constructed value. + */ + protected FieldValue createByReflection(Object arg) { + Class<?> valClass = getValueClass(); + if (valClass != null) { + Pair<Class<?>, Class<?>> key = new Pair<>(valClass, arg.getClass()); + Constructor<?> cstr = constructorCache.get(key); + try { + if (cstr == null) { + cstr = valClass.getConstructor(key.getSecond()); + constructorCache.put(key, cstr); + } + return (FieldValue)cstr.newInstance(arg); + } catch (ReflectiveOperationException e) { + // Only rethrow exceptions coming from the underlying FieldValue constructor. + if (e instanceof InvocationTargetException) { + throw new IllegalArgumentException(e.getCause().getMessage(), e.getCause()); + } + } + } + return null; + } + + /** + * Creates a new FieldValue of this type, with the given value. + * + * @param arg the value that the new FieldValue shall have. + * @return A new FieldValue of this type, with the given value. + */ + public FieldValue createFieldValue(Object arg) { + if (arg == null) { + return createFieldValue(); + } + FieldValue val = createByReflection(arg); + if (val == null) { + val = createFieldValue(); + if (val != null) { + val.assign(arg); + } + } + return val; + } + + public abstract Class getValueClass(); + + public abstract boolean isValueCompatible(FieldValue value); + + public final boolean isAssignableFrom(DataType dataType) { + // TODO: Reverse this so that isValueCompatible() uses this instead. + return isValueCompatible(dataType.createFieldValue()); + } + + /** + * Returns an array datatype, where the array elements are of the given type + * + * @param type the type to create an array of + * @return the array data type + */ + public static ArrayDataType getArray(DataType type) { + return new ArrayDataType(type); + } + + /** + * Returns a map datatype + * + * @param key the key type + * @param value the value type + * @return the map data type + */ + public static MapDataType getMap(DataType key, DataType value) { + return new MapDataType(key, value); + } + + /** + * Returns a weighted set datatype, where the elements are of the given type + * + * @param type the type to create a weighted set of + * @return the weighted set data type + */ + public static WeightedSetDataType getWeightedSet(DataType type) { + return getWeightedSet(type, false, false); + } + + /** + * Returns a weighted set datatype, where the elements are of the given type, and which supports the properties + * createIfNonExistent and removeIfZero + * + * @param type the type to create a weighted set of + * @param createIfNonExistent whether the type has createIfNonExistent + * @param removeIfZero whether the type has removeIfZero + * @return the weighted set data type + */ + public static WeightedSetDataType getWeightedSet(DataType type, boolean createIfNonExistent, boolean removeIfZero) { + return new WeightedSetDataType(type, createIfNonExistent, removeIfZero); + } + + public java.lang.String getName() { + return name; + } + + /** + * Sets the name of this data type. WARNING! Do not use! + * + * @param name the name of this data type. + */ + protected void setName(String name) { + this.name = name; + } + + public int getId() { + return dataTypeId; + } + + /** + * Sets the ID of this data type. WARNING! Do not use! + * + * @param id the ID of this data type. + */ + public void setId(int id) { + dataTypeId = id; + } + + /** + * Registeres this type in the given document manager. + * + * @param manager the DocumentTypeManager to register in. + */ + public final void register(DocumentTypeManager manager) { + register(manager, new LinkedList<>()); + } + + protected void register(DocumentTypeManager manager, List<DataType> seenTypes) { + manager.registerSingleType(this); + } + + public int hashCode() { + return name.hashCode(); + } + + public boolean equals(Object other) { + if (!(other instanceof DataType)) { + return false; + } + DataType type = (DataType)other; + return (name.equals(type.name) && dataTypeId == type.dataTypeId); + } + + public java.lang.String toString() { + return "datatype " + name + " (code: " + dataTypeId + ")"; + } + + public int getCode() { + return dataTypeId; + } + + /** + * Creates a field path from the given field path string. + * + * @param fieldPathString a string containing the field path + * @return Returns a valid field path, parsed from the string + */ + public FieldPath buildFieldPath(String fieldPathString) { + if (fieldPathString.length() > 0) { + throw new IllegalArgumentException( + "Datatype " + toString() + " does not support further recursive structure: " + fieldPathString); + } + return new FieldPath(); + } + + /** + * Returns the primitive datatype associated with this datatype, i.e. the type itself if this is a + * PrimitiveDataType, the nested type if this is a CollectionDataType or null for all other cases + * + * @return primitive data type, or null + */ + public PrimitiveDataType getPrimitiveType() { + return null; + } + + @Override + public void visitMembers(ObjectVisitor visitor) { + super.visitMembers(visitor); + visitor.visit("name", name); + visitor.visit("id", dataTypeId); + } + + /** + * Utility function for parsing field paths. + */ + static String skipDotInString(String remaining, int endPos) { + if (remaining.length() < endPos + 2) { + return ""; + } else if (remaining.charAt(endPos + 1) == '.') { + return remaining.substring(endPos + 2); + } else { + return remaining.substring(endPos + 1); + } + } + + @Override + public int compareTo(DataType dataType) { + return Integer.valueOf(dataTypeId).compareTo(dataType.dataTypeId); + } + + /** Returns whether this is a multivalue type, i.e either a CollectionDataType or a MapDataType */ + public boolean isMultivalue() { return false; } + +} diff --git a/document/src/main/java/com/yahoo/document/DataTypeName.java b/document/src/main/java/com/yahoo/document/DataTypeName.java new file mode 100644 index 00000000000..b7d5be5c5c0 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DataTypeName.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.text.Lowercase; +import com.yahoo.text.Utf8Array; +import com.yahoo.text.Utf8String; + +import java.io.Serializable; + +/** + * A full document type name, consisting of a <i>name</i> and a <i>version</i>. The name is case insensitive, and the + * version must be a positive integer or 0. This is a <i>value object</i>. + * + * @author bratseth + */ +public final class DataTypeName implements Serializable { + + private final Utf8String name; + + /** + * Creates a document name from a string of the form "name" + * + * @param name The name string to parse. + * @throws NumberFormatException if the version part of the name is present but is not a number + */ + public DataTypeName(String name) { + this.name = new Utf8String(name); + } + public DataTypeName(Utf8Array name) { + this.name = new Utf8String(name); + } + public DataTypeName(Utf8String name) { + this.name = new Utf8String(name); + } + + public String getName() { return name.toString(); } + + @Override + public String toString() { return name.toString(); } + + @Override + public int hashCode() { return name.hashCode(); } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof DataTypeName)) return false; + DataTypeName datatype = (DataTypeName)obj; + return this.name.equals(datatype.name); + } + +} diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java new file mode 100644 index 00000000000..740c91c5c1b --- /dev/null +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -0,0 +1,397 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.serialization.*; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.vespa.objects.BufferSerializer; +import com.yahoo.vespa.objects.Ids; +import com.yahoo.vespa.objects.Serializer; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Iterator; +import java.util.Map; + +/** + * A document is an identifiable + * set of value bindings of a {@link DocumentType document type}. + * A document represents an instance of some entity of interest + * in an application, like an article, a web document, a product, etc. + * + * Deprecation: Try to use document set and get methods only with FieldValue types, + * not with primitive types. Support for direct access to primitive types will + * be removed soon. + * + * @author <a href="bratseth@yahoo-inc.com">Jon S Bratseth</a> + * @author <a href="einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class Document extends StructuredFieldValue { + + public static final int classId = registerClass(Ids.document + 3, Document.class); + public static final short SERIALIZED_VERSION = 8; + private DocumentId docId; + private Struct header; + private Struct body; + private Long lastModified = null; + + /** + * Create a document with the given document type and identifier. + * @param docType DocumentType to use for creation + * @param id The id for this document + */ + public Document(DocumentType docType, String id) { + this(docType, new DocumentId(id)); + } + + /** + * Create a document with the given document type and identifier. + * @param docType DocumentType to use for creation + * @param id The id for this document + */ + public Document(DocumentType docType, DocumentId id) { + super(docType); + setNewType(docType); + internalSetId(id, docType); + } + + /** + * Creates a document that is a shallow copy of another. + * + * @param doc The document to copy. + */ + public Document(Document doc) { + this(doc.getDataType(), doc.getId()); + header = doc.header; + body = doc.body; + lastModified = doc.lastModified; + } + + /** + * + * @param reader The deserializer to use for creating this document + */ + public Document(DocumentReader reader) { + super(null); + reader.read(this); + } + + public DocumentId getId() { return docId; } + public void setId(DocumentId id) { internalSetId(id, getDataType()); } + private void internalSetId(DocumentId id, DocumentType docType) { + if (id != null && id.hasDocType() && docType != null && !id.getDocType().equals(docType.getName())) { + throw new IllegalArgumentException("Trying to set a document id (type " + id.getDocType() + + ") that don't match the document type (" + getDataType().getName() + ")."); + } + docId = id; + } + + public Struct getHeader() { return header; } + public Struct getBody() { return body; } + + @Override + public void assign(Object o) { + throw new IllegalArgumentException("Assign not implemented for " + getClass() + " objects"); + } + + @Override + public Document clone() { + Document doc = (Document) super.clone(); + doc.docId = docId.clone(); + doc.header = header.clone(); + doc.body = body.clone(); + return doc; + } + + private void setNewType(DocumentType type) { + header = type.getHeaderType().createFieldValue(); + body = type.getBodyType().createFieldValue(); + } + + public void setDataType(DataType type) { + if (docId != null && docId.hasDocType() && !docId.getDocType().equals(type.getName())) { + throw new IllegalArgumentException("Trying to set a document type (" + type.getName() + + ") that don't match the document id (" + docId + ")."); + } + super.setDataType(type); + setNewType((DocumentType)type); + } + + public int getSerializedSize() throws SerializationException { + DocumentSerializer data = DocumentSerializerFactory.create42(new GrowableByteBuffer(64 * 1024, 2.0f)); + data.write(this); + return data.getBuf().position(); + } + + /** + * This is an approximation of serialized size. We just set it to 4096 as a definition of a medium document. + * @return Approximate size of document (4096) + */ + public final int getApproxSize() { return 4096; } + + public void serialize(OutputStream out) throws SerializationException { + DocumentSerializer writer = DocumentSerializerFactory.create42(new GrowableByteBuffer(64 * 1024, 2.0f)); + writer.write(this); + GrowableByteBuffer data = writer.getBuf(); + byte[] array; + if (data.hasArray()) { + //just get the array + array = data.array(); + } else { + //copy the bytebuffer into the array + array = new byte[data.position()]; + int endPos = data.position(); + data.position(0); + data.get(array); + data.position(endPos); + } + try { + out.write(array, 0, data.position()); + } catch (IOException ioe) { + throw new SerializationException(ioe); + } + } + + public static Document createDocument(DocumentReader buffer) { + return new Document(buffer); + } + + @Override + public Field getField(String fieldName) { + Field field = header.getField(fieldName); + if (field == null) { + field = body.getField(fieldName); + } + if (field == null) { + for(DocumentType parent : getDataType().getInheritedTypes()) { + field = parent.getField(fieldName); + if (field != null) { + break; + } + } + } + return field; + } + + @Override + public FieldValue getFieldValue(Field field) { + if (field.isHeader()) { + return header.getFieldValue(field); + } else { + return body.getFieldValue(field); + } + } + + @Override + protected void doSetFieldValue(Field field, FieldValue value) { + if (field.isHeader()) { + header.setFieldValue(field, value); + } else { + body.setFieldValue(field, value); + } + } + + @Override + public FieldValue removeFieldValue(Field field) { + if (field.isHeader()) { + return header.removeFieldValue(field); + } else { + return body.removeFieldValue(field); + } + } + + @Override + public void clear() { + header.clear(); + body.clear(); + } + + @Override + public Iterator<Map.Entry<Field, FieldValue>> iterator() { + return new Iterator<Map.Entry<Field, FieldValue>>() { + + private Iterator<Map.Entry<Field, FieldValue>> headerIt = header.iterator(); + private Iterator<Map.Entry<Field, FieldValue>> bodyIt = body.iterator(); + + public boolean hasNext() { + if (headerIt != null) { + if (headerIt.hasNext()) { + return true; + } else { + headerIt = null; + } + } + return bodyIt.hasNext(); + } + + public Map.Entry<Field, FieldValue> next() { + return (headerIt == null ? bodyIt.next() : headerIt.next()); + } + + public void remove() { + if (headerIt == null) { + bodyIt.remove(); + } else { + headerIt.remove(); + } + } + }; + } + + public String toString() { + return "document '" + String.valueOf(docId) + "' of type '" + getDataType().getName() + "'"; + } + + public String toXML(String indent) { + XmlStream xml = new XmlStream(); + xml.setIndent(indent); + xml.beginTag("document"); + printXml(xml); + xml.endTag(); + return xml.toString(); + } + + /** + * Get XML representation of the document root and its children, contained + * within a <document></document> tag. + * @return XML representation of document + */ + public String toXml() { + return toXML(" "); + } + + public void printXml(XmlStream xml) { + XmlSerializationHelper.printDocumentXml(this, xml); + } + + /** Returns true if the argument is a document which has the same set of values */ + public boolean equals(Object o) { + if (!(o instanceof Document)) return false; + Document other = (Document) o; + return (super.equals(o) && docId.equals(other.docId) && + header.equals(other.header) && body.equals(other.body)); + } + + @Override + public int hashCode() { + return 31 * super.hashCode() + (docId != null ? docId.hashCode() : 0); + } + + /** + * Returns the last modified time of this Document, when stored in persistent storage. This is typically set by the + * library that retrieves the Document from persistent storage. + * + * This variable doesn't really belong in document. It is used when retrieving docblocks of documents to be able to + * see when documents was last modified in VDS, without having to add modified times separate in the API. + * + * NOTE: This is a transient field, and will not be serialized with a Document (will be null after deserialization). + * + * @return the last modified time of this Document (in milliseconds), or null if unset + */ + public Long getLastModified() { + return lastModified; + } + + /** + * Sets the last modified time of this Document. This is typically set by the library that retrieves the + * Document from persistent storage, and should not be set by arbitrary clients. NOTE: This is a + * transient field, and will not be serialized with a Document (will be null after deserialization). + * + * @param lastModified the last modified time of this Document (in milliseconds) + */ + public void setLastModified(Long lastModified) { + this.lastModified = lastModified; + } + + public void onSerialize(Serializer data) throws SerializationException { + serialize((DocumentWriter)data); + } + + public void serializeHeader(Serializer data) throws SerializationException { + if (data instanceof DocumentWriter) { + if (data instanceof VespaDocumentSerializer42) { + ((VespaDocumentSerializer42)data).setHeaderOnly(true); + } + serialize((DocumentWriter)data); + } else if (data instanceof BufferSerializer) { + serialize(DocumentSerializerFactory.create42(((BufferSerializer) data).getBuf(), true)); + } else { + DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true); + serialize(fw); + data.put(null, fw.getBuf().getByteBuffer()); + } + } + + public void serializeBody(Serializer data) throws SerializationException { + if (getBody().getFieldCount() > 0) { + if (data instanceof FieldWriter) { + getBody().serialize(new Field("body", getBody().getDataType()), (FieldWriter) data); + } else if (data instanceof BufferSerializer) { + getBody().serialize(new Field("body", getBody().getDataType()), DocumentSerializerFactory.create42(((BufferSerializer) data).getBuf())); + } else { + DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer()); + getBody().serialize(new Field("body", getBody().getDataType()), fw); + data.put(null, fw.getBuf().getByteBuffer()); + } + } + } + + @Override + public DocumentType getDataType() { + return (DocumentType)super.getDataType(); + } + + @Override + public int getFieldCount() { + return header.getFieldCount() + body.getFieldCount(); + } + + public void serialize(DocumentWriter writer) { + writer.write(this); + } + + public void deserialize(DocumentReader reader) { + reader.read(this); + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + /* (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + + //types are equal, this must be of this type + Document otherValue = (Document) fieldValue; + comp = getId().compareTo(otherValue.getId()); + + if (comp != 0) { + return comp; + } + + comp = header.compareTo(otherValue.header); + + if (comp != 0) { + return comp; + } + + comp = body.compareTo(otherValue.body); + return comp; + } +} diff --git a/document/src/main/java/com/yahoo/document/DocumentCalculator.java b/document/src/main/java/com/yahoo/document/DocumentCalculator.java new file mode 100644 index 00000000000..312f72e432d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentCalculator.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.select.Context; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.select.parser.SelectInput; +import com.yahoo.document.select.parser.SelectParser; +import com.yahoo.document.select.rule.ComparisonNode; + +import java.util.Map; + +/** + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class DocumentCalculator { + + private ComparisonNode comparison; + + public DocumentCalculator(String expression) throws ParseException { + SelectParser parser = new SelectParser(new SelectInput(expression + " == 0")); + comparison = (ComparisonNode)parser.expression(); + } + + public Number evaluate(Document doc, Map<String, Object> variables) { + Context context = new Context(new DocumentPut(doc)); + context.setVariables(variables); + + try { + Object o = comparison.getLHS().evaluate(context); + + if (Double.isInfinite(((Number)o).doubleValue())) { + throw new IllegalArgumentException("Expression evaluated to an infinite number"); + } + return ((Number)o).doubleValue(); + } catch (ArithmeticException e) { + throw new IllegalArgumentException("Arithmetic exception " + e.getMessage(), e); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/DocumentId.java b/document/src/main/java/com/yahoo/document/DocumentId.java new file mode 100644 index 00000000000..59650ea23f0 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentId.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.idstring.IdString; +import com.yahoo.document.serialization.*; +import com.yahoo.vespa.objects.Deserializer; +import com.yahoo.vespa.objects.Identifiable; +import com.yahoo.vespa.objects.Serializer; + +import java.io.Serializable; + +/** + * The id of a document + */ +public class DocumentId extends Identifiable implements Serializable { + + private IdString id; + private GlobalId globalId; + + /** + * Constructor used for deserialization. + */ + public DocumentId(Deserializer buf) { + deserialize(buf); + } + + /** + * Constructor. This constructor is used if the DocumentId is used outside of a Document object, but we have the + * URI. + * + * @param id Associate with this URI, storage address etc. is not applicable. + */ + public DocumentId(String id) { + if (id == null) { + throw new IllegalArgumentException("Cannot create DocumentId from null id."); + } + this.id = IdString.createIdString(id); + globalId = null; + } + + public DocumentId(IdString id) { + this.id = id; + globalId = null; + } + + @Override + public DocumentId clone() { + DocumentId docId = (DocumentId)super.clone(); + return docId; + } + + public void setId(IdString id) { + this.id = id; + } + + public IdString getScheme() { + return id; + } + + public byte[] getGlobalId() { + if (globalId == null) { + globalId = new GlobalId(id); + } + return globalId.getRawId(); + } + + public int compareTo(Object o) { + DocumentId cmp = (DocumentId)o; + return id.toString().compareTo(cmp.id.toString()); + } + + public boolean equals(Object o) { + return o instanceof DocumentId && id.equals(((DocumentId)o).id); + } + + public int hashCode() { + return id.hashCode(); + } + + public String toString() { + return id.toString(); + } + + @Override + public void onSerialize(Serializer target) throws SerializationException { + if (target instanceof DocumentWriter) { + ((DocumentWriter)target).write(this); + } else { + target.put(null, id.toString()); + } + } + + + public void onDeserialize(Deserializer data) throws DeserializationException { + if (data instanceof DocumentReader) { + id = ((DocumentReader)data).readDocumentId().getScheme(); + } else { + id = IdString.createIdString(data.getString(null)); + } + } + + public boolean hasDocType() { + return id.hasDocType(); + } + + public String getDocType() { + return id.getDocType(); + } +} diff --git a/document/src/main/java/com/yahoo/document/DocumentOperation.java b/document/src/main/java/com/yahoo/document/DocumentOperation.java new file mode 100644 index 00000000000..250e780e65a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentOperation.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.google.common.annotations.Beta; + +/** + * Base class for "document operations". + * These include "put" (DocumentPut), "update" (DocumentUpdate), and "remove" (DocumentRemove). + * Historically, put operations were represented by the Document class alone, + * but since it doesn't make much sense to put a *test and set* condition in Document, + * a more uniform interface for document operations was needed. + * + * @author Vegard Sjonfjell + */ +public abstract class DocumentOperation { + + private TestAndSetCondition condition = TestAndSetCondition.NOT_PRESENT_CONDITION; + + public abstract DocumentId getId(); + + public void setCondition(TestAndSetCondition condition) { + this.condition = condition; + } + + public TestAndSetCondition getCondition() { + return condition; + } + + protected DocumentOperation() {} + + /** + * Copy constructor + * @param other DocumentOperation to copy + */ + protected DocumentOperation(DocumentOperation other) { + this.condition = other.condition; + } + +} diff --git a/document/src/main/java/com/yahoo/document/DocumentPut.java b/document/src/main/java/com/yahoo/document/DocumentPut.java new file mode 100644 index 00000000000..f02d1e6d6d8 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentPut.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +/** + * @author Vegard Sjonfjell + */ +public class DocumentPut extends DocumentOperation { + + private final Document document; + + public DocumentPut(Document document) { + this.document = document; + } + + public DocumentPut(DocumentType docType, DocumentId docId) { + this.document = new Document(docType, docId); + } + + public DocumentPut(DocumentType docType, String docId) { + this.document = new Document(docType, docId); + } + + public Document getDocument() { + return document; + } + + public DocumentId getId() { + return document.getId(); + } + + /** + * Copy constructor + * @param other DocumentPut to copy + */ + public DocumentPut(DocumentPut other) { + super(other); + this.document = new Document(other.getDocument()); + } + + /** + * Base this DocumentPut on another, but use newDocument as the Document. + */ + public DocumentPut(DocumentPut other, Document newDocument) { + super(other); + this.document = newDocument; + } + +} diff --git a/document/src/main/java/com/yahoo/document/DocumentRemove.java b/document/src/main/java/com/yahoo/document/DocumentRemove.java new file mode 100644 index 00000000000..8d4f37d5583 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentRemove.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class DocumentRemove extends DocumentOperation { + + private final DocumentId docId; + + public DocumentRemove(DocumentId docId) { this.docId = docId; } + + @Override + public DocumentId getId() { return docId; } + + @Override + public String toString() { + return "DocumentRemove '" + docId + "'"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DocumentRemove)) return false; + DocumentRemove that = (DocumentRemove) o; + if (!docId.equals(that.docId)) return false; + return true; + } + + @Override + public int hashCode() { + return docId.hashCode(); + } + +} diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java new file mode 100755 index 00000000000..db38228489d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -0,0 +1,476 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.serialization.DocumentWriter; +import com.yahoo.vespa.objects.Ids; +import com.yahoo.vespa.objects.ObjectVisitor; +import com.yahoo.vespa.objects.Serializer; + +import java.util.*; + +/** + * <p>A document definition is a list of fields. Documents may inherit other documents, + * implicitly acquiring their fields as it's own. If a document is not set to inherit + * any document, it will always inherit the document "document.0".</p> + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class DocumentType extends StructuredDataType { + + public static final int classId = registerClass(Ids.document + 58, DocumentType.class); + private StructDataType headerType; + private StructDataType bodyType; + private List<DocumentType> inherits = new ArrayList<DocumentType>(1); + + /** + * Creates a new document type and registers it with the document type manager. + * This will be created as version 0 of this document type. + * Implicitly registers this with the document type manager. + * The document type id will be generated as a hash from the document type name. + * + * @param name The name of the new document type + */ + public DocumentType(String name) { + this(name, new StructDataType(name + ".header"), + new StructDataType(name + ".body")); + } + + /** + * Creates a new document type and registers it with the document type manager. + * Implicitly registers this with the document type manager. + * The document type id will be generated as a hash from the document type name. + * + * @param name The name of the new document type + * @param headerType The type of the header struct + * @param bodyType The type of the body struct + */ + public DocumentType(String name, StructDataType headerType, StructDataType bodyType) { + super(name); + this.headerType = headerType; + this.bodyType = bodyType; + } + + @Override + public DocumentType clone() { + DocumentType type = (DocumentType) super.clone(); + type.headerType = headerType.clone(); + type.bodyType = bodyType.clone(); + type.inherits = new ArrayList<>(inherits.size()); + for (DocumentType inherited : inherits) { + type.inherits.add(inherited); + } + return type; + } + + @Override + public Document createFieldValue() { + return new Document(this, (DocumentId) null); + } + + @Override + public Class getValueClass() { + return Document.class; + } + + @Override + public boolean isValueCompatible(FieldValue value) { + if (!(value instanceof Document)) { + return false; + } + Document doc = (Document) value; + if (doc.getDataType().inherits(this)) { + //the value is of this type; or the supertype of the value is of this type, etc.... + return true; + } + return false; + } + + public StructDataType getHeaderType() { + return headerType; + } + + public StructDataType getBodyType() { + return bodyType; + } + + @Override + protected void register(DocumentTypeManager manager, List<DataType> seenTypes) { + seenTypes.add(this); + for (DocumentType type : getInheritedTypes()) { + if (!seenTypes.contains(type)) { + type.register(manager, seenTypes); + } + } + // Get parent fields into fields specified in this type + StructDataType header = headerType.clone(); + StructDataType body = bodyType.clone(); + + header.clearFields(); + body.clearFields(); + + for (Field field : fieldSet()) { + (field.isHeader() ? header : body).addField(field); + } + headerType.assign(header); + bodyType.assign(body); + + if (!seenTypes.contains(headerType)) { + headerType.register(manager, seenTypes); + } + if (!seenTypes.contains(bodyType)) { + bodyType.register(manager, seenTypes); + } + manager.registerSingleType(this); + } + + /** + * Check if this document type has the given name, + * or inherits from a type with that name. + */ + public boolean isA(String docTypeName) { + if (getName().equalsIgnoreCase(docTypeName)) { + return true; + } + for (DocumentType parent : inherits) { + if (parent.isA(docTypeName)) { + return true; + } + } + return false; + } + + /** + * Adds an field that can be used with this document type. + * + * @param field the field to add + */ + public void addField(Field field) { + if (isRegistered()) { + throw new IllegalStateException("You cannot add fields to a document type that is already registered."); + } + StructDataType struct = (field.isHeader() ? headerType : bodyType); + struct.addField(field); + } + + /** + * Adds a new body field to this document type and returns the new field object + * + * @param name The name of the field to add + * @param type The datatype of the field to add + * @return The field created + * TODO Fix searchdefinition so that exception can be thrown if filed is already registerd. + */ + public Field addField(String name, DataType type) { + if (isRegistered()) { + throw new IllegalStateException("You cannot add fields to a document type that is already registered."); + } + Field field = new Field(name, type, false); + bodyType.addField(field); + return field; + } + + /** + * Adds a new header field to this document type and returns the new field object + * + * @param name The name of the field to add + * @param type The datatype of the field to add + * @return The field created + * TODO Fix searchdefinition so that exception can be thrown if filed is already registerd + */ + public Field addHeaderField(String name, DataType type) { + if (isRegistered()) { + throw new IllegalStateException("You cannot add fields to a document type that is already registered."); + } + Field field = new Field(name, type, true); + headerType.addField(field); + return field; + } + + /** + * Adds a document to the inherited document types of this. + * If this type is already directly inherited, nothing is done + * + * @param type An already DocumentType object. + */ + public void inherit(DocumentType type) { + //TODO: There is also a check like the following in SDDocumentType addField(), try to move that to this class' addField() to get it proper, + // as this method is called only when the doc types are exported. + verifyTypeConsistency(type); + if (isRegistered()) { + throw new IllegalStateException("You cannot add inheritance to a document type that is already registered."); + } + if (type == null) { + throw new IllegalArgumentException("The document type cannot be null in inherit()"); + } + + // If it inherits the exact same type + if (inherits.contains(type)) return; + + // If we inherit a type, don't inherit the supertype + if (inherits.size() == 1 && inherits.get(0).getDataTypeName().equals(new DataTypeName("document"))) { + inherits.clear(); + } + + inherits.add(type); + } + + /** + * Fail if the subtype changes the type of any equally named field. + * + * @param superType The supertype to verify against + * TODO Add strict type checking no duplicate fields are allowed + */ + private void verifyTypeConsistency(DocumentType superType) { + for (Field f : fieldSet()) { + Field supField = superType.getField(f.getName()); + if (supField != null) { + if (!f.getDataType().equals(supField.getDataType())) { + throw new IllegalArgumentException("Inheritance type mismatch: field \"" + f.getName() + + "\" in datatype \"" + getName() + "\"" + + " must have same datatype as in parent document type \"" + superType.getName() + "\""); + } + } + } + } + + /** + * Returns the DocumentNames which are directly inherited by this + * as a read-only collection. + * If this document type does not explicitly inherit anything, the list will + * contain the root type 'Document' + * + * @return a read-only list iterator containing the name Strings of the directly + * inherited document types of this + */ + public Collection<DocumentType> getInheritedTypes() { + return Collections.unmodifiableCollection(inherits); + } + + public ListIterator<DataTypeName> inheritedIterator() { + List<DataTypeName> names = new ArrayList<>(inherits.size()); + for (DocumentType type : inherits) { + names.add(type.getDataTypeName()); + } + return ImmutableList.copyOf(names).listIterator(); + } + + /** + * Return whether this document type inherits the given document type. + * + * @param superType The documenttype to check if it inherits. + * @return true if it inherits the superType, false if not + */ + public boolean inherits(DocumentType superType) { + if (equals(superType)) return true; + for (DocumentType type : inherits) { + if (type.inherits(superType)) return true; + } + return false; + } + + /** + * Gets the field matching a given name. + * + * @param name The name of a field. + * @return Returns the matching field, or null if not found. + */ + public Field getField(String name) { + Field field = headerType.getField(name); + if (field == null) { + field = bodyType.getField(name); + } + if (field == null && !isRegistered()) { + for (DocumentType inheritedType : inherits) { + field = inheritedType.getField(name); + if (field != null) break; + } + } + return field; + } + + @Override + public Field getField(int id) { + Field field = headerType.getField(id); + if (field == null) { + field = bodyType.getField(id); + } + if (field == null && !isRegistered()) { + for (DocumentType inheritedType : inherits) { + field = inheritedType.getField(id); + if (field != null) break; + } + } + return field; + } + + /** + * Returns whether this type defines the given field name + * + * @param name The name of the field to check if it has + * @return True if there is a field with the given name. + */ + public boolean hasField(String name) { + return getField(name) != null; + } + + //@Override + + + public int getFieldCount() { + return headerType.getFieldCount() + bodyType.getFieldCount(); + } + + /** + * Gets the field matching a given ID. + * + * @param id The ID of a field. + * @param version The serialization version of the document. + * @return Returns the matching field, or null if not found. + */ + public Field getField(Integer id, int version) { + Field field = headerType.getField(id, version); + if (field == null) { + field = bodyType.getField(id, version); + } + if (field == null && !isRegistered()) { + for (DocumentType inheritedType : inherits) { + field = inheritedType.getField(id, version); + if (field != null) break; + } + } + return field; + } + + /** + * Removes an field from the DocumentType. + * + * @param name The name of the field. + * @return The field that was removed or null if it did not exist. + */ + public Field removeField(String name) { + if (isRegistered()) { + throw new IllegalStateException("You cannot remove fields from a document type that is already registered."); + } + Field field = headerType.removeField(name); + if (field == null) { + field = bodyType.removeField(name); + } + if (field == null) { + for (DocumentType inheritedType : inherits) { + field = inheritedType.removeField(name); + if (field != null) break; + } + } + return field; + } + + public Collection<Field> getFields() { + Collection<Field> collection = new LinkedList<>(); + + for (DocumentType type : inherits) { + collection.addAll(type.getFields()); + } + + collection.addAll(headerType.getFields()); + collection.addAll(bodyType.getFields()); + return ImmutableList.copyOf(collection); + } + + /** + * <p>Returns an ordered set snapshot of all fields of this documenttype, + * <i>except the fields of Document</i>. + * Only the overridden version will be returned for overridden fields.</p> + * + * <p>The fields of a document type has a well-defined order which is + * exhibited in this set: + * - Fields come in the order defined in the document type definition. + * - The fields defined in inherited types come before those in + * the document type itself. + * - When a field in an inherited type is overridden, the value is overridden, + * but not the ordering. + * </p> + * + * @return an unmodifiable snapshot of the fields in this type + */ + public Set<Field> fieldSet() { + Map<String, Field> map = new LinkedHashMap<String, Field>(); + for (Field field : getFields()) { // Uniqify on field name + map.put(field.getName(), field); + } + return ImmutableSet.copyOf(map.values()); + } + + /** + * Returns an iterator over all fields in this documenttype + * + * @return An iterator for iterating the fields in this documenttype. + */ + public Iterator<Field> fieldIteratorThisTypeOnly() { + return new Iterator<Field>() { + Iterator<Field> headerIt = headerType.getFields().iterator(); + Iterator<Field> bodyIt = bodyType.getFields().iterator(); + + public boolean hasNext() { + if (headerIt != null) { + if (headerIt.hasNext()) return true; + headerIt = null; + } + return bodyIt.hasNext(); + } + + public Field next() { + return (headerIt != null ? headerIt.next() : bodyIt.next()); + } + + + public void remove() { + if (headerIt != null) { + headerIt.remove(); + } else { + bodyIt.remove(); + } + } + }; + } + + public boolean equals(Object o) { + if (!(o instanceof DocumentType)) return false; + DocumentType other = (DocumentType) o; + // Ignore whether one of them have added inheritance to super Document.0 type + if (super.equals(o) && headerType.equals(other.headerType) && + bodyType.equals(other.bodyType)) { + if ((inherits.size() > 1 || other.inherits.size() > 1) || + (inherits.size() == 1 && other.inherits.size() == 1)) { + return inherits.equals(other.inherits); + } + return !(((inherits.size() == 1) && !inherits.get(0).getDataTypeName().equals(new DataTypeName("document"))) + || ((other.inherits.size() == 1) && !other.inherits.get(0).getDataTypeName().equals(new DataTypeName("document")))); + } + return false; + } + + public int hashCode() { + return super.hashCode() + headerType.hashCode() + bodyType.hashCode() + inherits.hashCode(); + } + + @Override + public void onSerialize(Serializer target) { + if (target instanceof DocumentWriter) { + ((DocumentWriter) target).write(this); + } + // TODO: what if it's not a DocumentWriter? + } + + + @Override + public void visitMembers(ObjectVisitor visitor) { + super.visitMembers(visitor); + visitor.visit("headertype", headerType); + visitor.visit("bodytype", bodyType); + visitor.visit("inherits", inherits); + } +} diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeId.java b/document/src/main/java/com/yahoo/document/DocumentTypeId.java new file mode 100644 index 00000000000..46e5040c998 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentTypeId.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +/** + * The id of a document type. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentTypeId { + private int id; + + public DocumentTypeId(int id) { + this.id = id; + } + + public int getId() { + return id; + } + + public boolean equals(Object o) { + if (!(o instanceof DocumentTypeId)) return false; + DocumentTypeId other = (DocumentTypeId) o; + return other.id == this.id; + } + + public int hashCode() { + return id; + } + + public String toString() { + return "" + id; + } +} diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java new file mode 100644 index 00000000000..0de7fe60500 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java @@ -0,0 +1,363 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.google.inject.Inject; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.annotation.AnnotationTypeRegistry; +import com.yahoo.document.annotation.AnnotationTypes; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentDeserializerFactory; +import com.yahoo.document.serialization.VespaDocumentDeserializer42; +import com.yahoo.io.GrowableByteBuffer; + +import java.lang.reflect.Modifier; +import java.util.*; +import java.util.logging.Logger; + +/** + * The DocumentTypeManager keeps track of the document types registered in + * the Vespa common repository. + * <p> + * The DocumentTypeManager is also responsible for registering a FieldValue + * factory for each data type a field can have. The Document object + * uses this factory to serialize and deserialize the various datatypes. + * The factory could also be used to expand the functionality of various + * datatypes, for instance displaying the data type in human-readable form + * or as XML. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class DocumentTypeManager { + + private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName()); + private ConfigSubscriber subscriber; + + private Map<Integer, DataType> dataTypes = new LinkedHashMap<>(); + private Map<DataTypeName, DocumentType> documentTypes = new LinkedHashMap<>(); + private AnnotationTypeRegistry annotationTypeRegistry = new AnnotationTypeRegistry(); + + public DocumentTypeManager() { + registerDefaultDataTypes(); + } + + @Inject + public DocumentTypeManager(DocumentmanagerConfig config) { + this(); + DocumentTypeManagerConfigurer.configureNewManager(config, this); + } + + public void assign(DocumentTypeManager other) { + dataTypes = other.dataTypes; + documentTypes = other.documentTypes; + annotationTypeRegistry = other.annotationTypeRegistry; + } + + public DocumentTypeManager configure(String configId) { + subscriber = DocumentTypeManagerConfigurer.configure(this, configId); + return this; + } + + private void registerDefaultDataTypes() { + DocumentType superDocType = DataType.DOCUMENT; + dataTypes.put(superDocType.getId(), superDocType); + documentTypes.put(superDocType.getDataTypeName(), superDocType); + + Class<? extends DataType> dataTypeClass = DataType.class; + for (java.lang.reflect.Field field : dataTypeClass.getFields()) { + if (Modifier.isStatic(field.getModifiers()) && Modifier.isFinal(field.getModifiers())) { + if (DataType.class.isAssignableFrom(field.getType())) { + try { + //these are all static final DataTypes listed in DataType: + DataType type = (DataType) field.get(null); + register(type); + } catch (IllegalAccessException e) { + //ignore + } + } + } + } + for (AnnotationType type : AnnotationTypes.ALL_TYPES) { + annotationTypeRegistry.register(type); + } + } + + public boolean hasDataType(String name) { + for (DataType type : dataTypes.values()) { + if (type.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; + } + + public boolean hasDataType(int code) { + return dataTypes.containsKey(code); + } + + public DataType getDataType(String name) { + List<DataType> foundTypes = new ArrayList<>(); + for (DataType type : dataTypes.values()) { + if (type.getName().equalsIgnoreCase(name)) { + foundTypes.add(type); + } + } + if (foundTypes.isEmpty()) { + throw new IllegalArgumentException("No datatype named " + name); + } else if (foundTypes.size() == 1) { + return foundTypes.get(0); + } else { + //the found types are probably documents or structs, sort them by type + Collections.sort(foundTypes, new Comparator<DataType>() { + public int compare(DataType first, DataType second) { + if (first instanceof StructuredDataType && !(second instanceof StructuredDataType)) { + return 1; + } else if (!(first instanceof StructuredDataType) && second instanceof StructuredDataType) { + return -1; + } else { + return 0; + } + } + }); + } + return foundTypes.get(0); + } + + public DataType getDataType(int code) { + DataType type = dataTypes.get(code); + if (type == null) { + StringBuilder types=new StringBuilder(); + for (Integer key : dataTypes.keySet()) { + types.append(key).append(" "); + } + throw new IllegalArgumentException("No datatype with code " + code + ". Registered type ids: " + types); + } else { + return type; + } + } + + DataType getDataTypeAndReturnTemporary(int code) { + if (hasDataType(code)) { + return getDataType(code); + } + return new TemporaryDataType(code); + } + + /** + * Register a data type of any sort, including document types. + * @param type The datatype to register + * TODO Give unique ids to document types + */ + public void register(DataType type) { + type.register(this); // Recursively walk through all nested types and call registerSingleType for each one + } + + /** + * Register a single datatype. Re-registering an existing, but equal, datatype is ok. + * @param type The datatype to register + */ + void registerSingleType(DataType type) { + if (dataTypes.containsKey(type.getId())) { + DataType existingType = dataTypes.get(type.getId()); + if (((type instanceof TemporaryDataType) || (type instanceof TemporaryStructuredDataType)) + && !((existingType instanceof TemporaryDataType) || (existingType instanceof TemporaryStructuredDataType))) { + //we're trying to register a temporary type over a permanent one, don't do that: + return; + } else if ((existingType == type || existingType.equals(type)) + && !(existingType instanceof TemporaryDataType) + && !(type instanceof TemporaryDataType) + && !(existingType instanceof TemporaryStructuredDataType) + && !(type instanceof TemporaryStructuredDataType)) { // Shortcut to improve speed + // Oki. Already registered. + return; + } else if (type instanceof DocumentType && dataTypes.get(type.getId()) instanceof DocumentType) { + /* + DocumentType newInstance = (DocumentType) type; + DocumentType oldInstance = (DocumentType) dataTypes.get(type.getId()); + TODO fix tests + */ + log.warning("Document type " + existingType + " is not equal to document type attempted registered " + type + + ", but have same name. OVERWRITING TYPE as many tests currently does this. " + + "Fix tests so we can throw exception here."); + //throw new IllegalStateException("Datatype " + existingType + " is not equal to datatype attempted registered " + // + type + ", but already uses id " + type.getId()); + } else if ((existingType instanceof TemporaryDataType) || (existingType instanceof TemporaryStructuredDataType)) { + //removing temporary type to be able to register correct type + dataTypes.remove(existingType.getId()); + } else { + throw new IllegalStateException("Datatype " + existingType + " is not equal to datatype attempted registered " + + type + ", but already uses id " + type.getId()); + } + } + + if (type instanceof DocumentType) { + DocumentType docType = (DocumentType) type; + if (docType.getInheritedTypes().size() == 0) { + docType.inherit(documentTypes.get(new DataTypeName("document"))); + } + documentTypes.put(docType.getDataTypeName(), docType); + } + dataTypes.put(type.getId(), type); + type.setRegistered(); + } + + /** + * Registers a document type. Typically called by someone + * importing the document types from the common Vespa repository. + * + * @param docType The document type to register. + * @return the previously registered type, or null if none was registered + */ + public DocumentType registerDocumentType(DocumentType docType) { + register(docType); + return docType; + } + + /** + * Gets a registered document. + * + * @param name the document name of the type + * @return returns the document type found, + * or null if there is no type with this name + */ + public DocumentType getDocumentType(DataTypeName name) { + return documentTypes.get(name); + } + + /** + * Returns a registered document type + * + * @param name the type name of the document type + * @return returns the document type having this name, or null if none + */ + public DocumentType getDocumentType(String name) { + return documentTypes.get(new DataTypeName(name)); + } + + final public Document createDocument(GrowableByteBuffer buf) { + DocumentDeserializer data = DocumentDeserializerFactory.create42(this, buf); + return new Document(data); + } + public Document createDocument(DocumentDeserializer data) { + return new Document(data); + } + + public Document createDocument(GrowableByteBuffer header, GrowableByteBuffer body) { + DocumentDeserializer data = DocumentDeserializerFactory.create42(this, header, body); + return new Document(data); + } + + /** + * A read only view of the registered data types + * @return collection of types + */ + public Collection<DataType> getDataTypes() { + return Collections.unmodifiableCollection(dataTypes.values()); + } + + /** + * A read only view of the registered document types + * @return map of types + */ + public Map<DataTypeName, DocumentType> getDocumentTypes() { + return Collections.unmodifiableMap(documentTypes); + } + + public Iterator<DocumentType> documentTypeIterator() { + return documentTypes.values().iterator(); + } + + /** + * Clears the DocumentTypeManager. After this operation, + * only the default document type and data types are available. + */ + public void clear() { + documentTypes.clear(); + dataTypes.clear(); + registerDefaultDataTypes(); + } + + public AnnotationTypeRegistry getAnnotationTypeRegistry() { + return annotationTypeRegistry; + } + + void replaceTemporaryTypes() { + for (DataType type : dataTypes.values()) { + List<DataType> seenStructs = new LinkedList<>(); + replaceTemporaryTypes(type, seenStructs); + } + } + + private void replaceTemporaryTypes(DataType type, List<DataType> seenStructs) { + if (type instanceof WeightedSetDataType) { + replaceTemporaryTypesInWeightedSet((WeightedSetDataType) type, seenStructs); + } else if (type instanceof MapDataType) { + replaceTemporaryTypesInMap((MapDataType) type, seenStructs); + } else if (type instanceof CollectionDataType) { + replaceTemporaryTypesInCollection((CollectionDataType) type, seenStructs); + } else if (type instanceof StructDataType) { + replaceTemporaryTypesInStruct((StructDataType) type, seenStructs); + } else if (type instanceof PrimitiveDataType) { + //OK + } else if (type instanceof AnnotationReferenceDataType) { + //OK + } else if (type instanceof DocumentType) { + //OK + } else if (type instanceof TemporaryDataType) { + throw new IllegalStateException("TemporaryDataType registered in DocumentTypeManager, BUG!!"); + } else { + log.warning("Don't know how to replace temporary data types in " + type); + } + } + + @SuppressWarnings("deprecation") + private void replaceTemporaryTypesInStruct(StructDataType structDataType, List<DataType> seenStructs) { + seenStructs.add(structDataType); + for (Field field : structDataType.getFieldsThisTypeOnly()) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof TemporaryDataType) { + field.setDataType(getDataType(fieldType.getCode())); + } else { + if (!seenStructs.contains(fieldType)) { + replaceTemporaryTypes(fieldType, seenStructs); + } + } + } + } + + private void replaceTemporaryTypesInCollection(CollectionDataType collectionDataType, List<DataType> seenStructs) { + if (collectionDataType.getNestedType() instanceof TemporaryDataType) { + collectionDataType.setNestedType(getDataType(collectionDataType.getNestedType().getCode())); + } else { + replaceTemporaryTypes(collectionDataType.getNestedType(), seenStructs); + } + } + + private void replaceTemporaryTypesInMap(MapDataType mapDataType, List<DataType> seenStructs) { + if (mapDataType.getValueType() instanceof TemporaryDataType) { + mapDataType.setValueType(getDataType(mapDataType.getValueType().getCode())); + } else { + replaceTemporaryTypes(mapDataType.getValueType(), seenStructs); + } + + if (mapDataType.getKeyType() instanceof TemporaryDataType) { + mapDataType.setKeyType(getDataType(mapDataType.getKeyType().getCode())); + } else { + replaceTemporaryTypes(mapDataType.getKeyType(), seenStructs); + } + } + + private void replaceTemporaryTypesInWeightedSet(WeightedSetDataType weightedSetDataType, List<DataType> seenStructs) { + if (weightedSetDataType.getNestedType() instanceof TemporaryDataType) { + weightedSetDataType.setNestedType(getDataType(weightedSetDataType.getNestedType().getCode())); + } else { + replaceTemporaryTypes(weightedSetDataType.getNestedType(), seenStructs); + } + } + + public void shutdown() { + if (subscriber!=null) subscriber.close(); + } +} diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java new file mode 100644 index 00000000000..a575fbfba2a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -0,0 +1,246 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.compress.CompressionType; +import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.log.LogLevel; +import java.util.ArrayList; +import java.util.logging.Logger; + +/** + * Configures the Vepsa document manager from a document id. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig>{ + + private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName()); + + private DocumentTypeManager managerToConfigure; + + public DocumentTypeManagerConfigurer(DocumentTypeManager manager) { + this.managerToConfigure = manager; + } + + private static CompressionConfig makeCompressionConfig(DocumentmanagerConfig.Datatype.Structtype cfg) { + return new CompressionConfig(toCompressorType(cfg.compresstype()), cfg.compresslevel(), + cfg.compressthreshold(), cfg.compressminsize()); + } + + + public static CompressionType toCompressorType(DocumentmanagerConfig.Datatype.Structtype.Compresstype.Enum value) { + switch (value) { + case NONE: return CompressionType.NONE; + case LZ4: return CompressionType.LZ4; + case UNCOMPRESSABLE: return CompressionType.INCOMPRESSIBLE; + } + throw new IllegalArgumentException("Compression type " + value + " is not supported"); + } + + /** + * <p>Makes the DocumentTypeManager subscribe on its config.</p> + * + * <p>Proper Vespa setups will use a config id which looks up the document manager config + * at the document server, but it is also possible to read config from a file containing + * a document manager configuration by using + * <code>file:path-to-document-manager.cfg</code>.</p> + * + * @param configId the config ID to use + */ + public static ConfigSubscriber configure(DocumentTypeManager manager, String configId) { + return new DocumentTypeManagerConfigurer(manager).configure(configId); + } + + public ConfigSubscriber configure(String configId) { + ConfigSubscriber subscriber = new ConfigSubscriber(); + subscriber.subscribe(this, DocumentmanagerConfig.class, configId); + return subscriber; + } + + static void configureNewManager(DocumentmanagerConfig config, DocumentTypeManager manager) { + if (config == null) { + return; + } + + setupAnnotationTypesWithoutPayloads(config, manager); + setupAnnotationRefTypes(config, manager); + + log.log(LogLevel.DEBUG, "Configuring document manager with " + config.datatype().size() + " data types."); + ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>(); + failed.addAll(config.datatype()); + int failCounter = 30; + while (!failed.isEmpty()) { + --failCounter; + ArrayList<DocumentmanagerConfig.Datatype> tmp = failed; + failed = new ArrayList<>(); + for (int i = 0; i < tmp.size(); i++) { + DocumentmanagerConfig.Datatype thisDataType = tmp.get(i); + int id = thisDataType.id(); + try { + for (Object o : thisDataType.arraytype()) { + DocumentmanagerConfig.Datatype.Arraytype array = (DocumentmanagerConfig.Datatype.Arraytype) o; + DataType nestedType = manager.getDataType(array.datatype()); + ArrayDataType type = new ArrayDataType(nestedType, id); + manager.register(type); + } + for (Object o : thisDataType.maptype()) { + DocumentmanagerConfig.Datatype.Maptype map = (DocumentmanagerConfig.Datatype.Maptype) o; + DataType keyType = manager.getDataType(map.keytype()); + DataType valType = manager.getDataType(map.valtype()); + MapDataType type = new MapDataType(keyType, valType, id); + manager.register(type); + } + for (Object o : thisDataType.weightedsettype()) { + DocumentmanagerConfig.Datatype.Weightedsettype wset = + (DocumentmanagerConfig.Datatype.Weightedsettype) o; + DataType nestedType = manager.getDataType(wset.datatype()); + WeightedSetDataType type = new WeightedSetDataType( + nestedType, wset.createifnonexistant(), wset.removeifzero(), id); + manager.register(type); + } + for (Object o : thisDataType.structtype()) { + DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o; + StructDataType type = new StructDataType(id, struct.name()); + + if (config.enablecompression()) { + CompressionConfig comp = makeCompressionConfig(struct); + type.setCompressionConfig(comp); + } + + for (Object j : struct.field()) { + DocumentmanagerConfig.Datatype.Structtype.Field field = + (DocumentmanagerConfig.Datatype.Structtype.Field) j; + DataType fieldType = (field.datatype() == id) + ? manager.getDataTypeAndReturnTemporary(field.datatype()) + : manager.getDataType(field.datatype()); + + if (field.id().size() == 1) { + type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true)); + } else { + type.addField(new Field(field.name(), fieldType, true)); + } + } + manager.register(type); + } + for (Object o : thisDataType.documenttype()) { + DocumentmanagerConfig.Datatype.Documenttype doc = (DocumentmanagerConfig.Datatype.Documenttype) o; + StructDataType header = (StructDataType) manager.getDataType(doc.headerstruct()); + StructDataType body = (StructDataType) manager.getDataType(doc.bodystruct()); + for (Field field : body.getFields()) { + field.setHeader(false); + } + DocumentType type = new DocumentType(doc.name(), header, body); + for (Object j : doc.inherits()) { + DocumentmanagerConfig.Datatype.Documenttype.Inherits parent = + (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j; + DataTypeName name = new DataTypeName(parent.name()); + DocumentType parentType = manager.getDocumentType(name); + if (parentType == null) { + throw new IllegalArgumentException("Could not find document type '" + name.toString() + "'."); + } + type.inherit(parentType); + } + manager.register(type); + } + } catch (IllegalArgumentException e) { + failed.add(thisDataType); + if (failCounter < 0) { + throw e; + } + } + } + } + addStructInheritance(config, manager); + addAnnotationTypePayloads(config, manager); + addAnnotationTypeInheritance(config, manager); + + manager.replaceTemporaryTypes(); + } + + public static DocumentTypeManager configureNewManager(DocumentmanagerConfig config) { + DocumentTypeManager manager = new DocumentTypeManager(); + if (config == null) { + return manager; + } + configureNewManager(config, manager); + return manager; + } + + /** + * Called by the configuration system to register document types based on documentmanager.cfg. + * + * @param config the instance representing config in documentmanager.cfg. + */ + @Override + public void configure(DocumentmanagerConfig config) { + DocumentTypeManager manager = configureNewManager(config); + int defaultTypeCount = new DocumentTypeManager().getDataTypes().size(); + if (this.managerToConfigure.getDataTypes().size() != defaultTypeCount) { + log.log(LogLevel.DEBUG, "Live document config overwritten with new config."); + } + managerToConfigure.assign(manager); + } + + private static void setupAnnotationRefTypes(DocumentmanagerConfig config, DocumentTypeManager manager) { + for (int i = 0; i < config.datatype().size(); i++) { + DocumentmanagerConfig.Datatype thisDataType = config.datatype(i); + int id = thisDataType.id(); + for (Object o : thisDataType.annotationreftype()) { + DocumentmanagerConfig.Datatype.Annotationreftype annRefType = (DocumentmanagerConfig.Datatype.Annotationreftype) o; + AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annRefType.annotation()); + if (annotationType == null) { + throw new IllegalArgumentException("Found reference to " + annRefType.annotation() + ", which does not exist!"); + } + AnnotationReferenceDataType type = new AnnotationReferenceDataType(annotationType, id); + manager.register(type); + } + } + } + + private static void setupAnnotationTypesWithoutPayloads(DocumentmanagerConfig config, DocumentTypeManager manager) { + for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) { + AnnotationType annotationType = new AnnotationType(annType.name(), annType.id()); + manager.getAnnotationTypeRegistry().register(annotationType); + } + } + + private static void addAnnotationTypePayloads(DocumentmanagerConfig config, DocumentTypeManager manager) { + for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) { + AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annType.id()); + DataType payload = manager.getDataType(annType.datatype()); + if (!payload.equals(DataType.NONE)) { + annotationType.setDataType(payload); + } + } + + } + + private static void addAnnotationTypeInheritance(DocumentmanagerConfig config, DocumentTypeManager manager) { + for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) { + if (annType.inherits().size() > 0) { + AnnotationType inheritedType = manager.getAnnotationTypeRegistry().getType(annType.inherits(0).id()); + AnnotationType type = manager.getAnnotationTypeRegistry().getType(annType.id()); + type.inherit(inheritedType); + } + } + } + + private static void addStructInheritance(DocumentmanagerConfig config, DocumentTypeManager manager) { + for (int i = 0; i < config.datatype().size(); i++) { + DocumentmanagerConfig.Datatype thisDataType = config.datatype(i); + int id = thisDataType.id(); + for (Object o : thisDataType.structtype()) { + DocumentmanagerConfig.Datatype.Structtype struct = (DocumentmanagerConfig.Datatype.Structtype) o; + StructDataType thisStruct = (StructDataType) manager.getDataType(id); + + for (DocumentmanagerConfig.Datatype.Structtype.Inherits parent : struct.inherits()) { + StructDataType parentStruct = (StructDataType) manager.getDataType(parent.name()); + thisStruct.inherit(parentStruct); + } + } + } + } +} diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java new file mode 100644 index 00000000000..0cfaa601b21 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java @@ -0,0 +1,415 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.document.serialization.DocumentUpdateReader; +import com.yahoo.document.serialization.DocumentUpdateWriter; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.io.GrowableByteBuffer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * <p>Specifies one or more field updates to a document.</p> <p>A document update contains a list of {@link + * com.yahoo.document.update.FieldUpdate field updates} for fields to be updated by this update. Each field update is + * applied atomically, but the entire document update is not. A document update can only contain one field update per + * field. To make multiple updates to the same field in the same document update, add multiple {@link + * com.yahoo.document.update.ValueUpdate value updates} to the same field update.</p> <p>To update a document and + * set a string field to a new value:</p> + * <pre> + * DocumentType musicType = DocumentTypeManager.getInstance().getDocumentType("music", 0); + * DocumentUpdate docUpdate = new DocumentUpdate(musicType, + * new DocumentId("doc:test:http://music.yahoo.com/")); + * FieldUpdate update = FieldUpdate.createAssign(musicType.getField("artist"), "lillbabs"); + * docUpdate.addFieldUpdate(update); + * </pre> + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @see com.yahoo.document.update.FieldUpdate + * @see com.yahoo.document.update.ValueUpdate + */ +public class DocumentUpdate extends DocumentOperation implements Iterable<FieldPathUpdate> { + + //see src/vespa/document/util/identifiableid.h + public static final int CLASSID = 0x1000 + 6; + + private DocumentId docId; + private List<FieldUpdate> fieldUpdates; + private List<FieldPathUpdate> fieldPathUpdates; + private DocumentType documentType; + private boolean createIfNonExistent; + + /** + * Creates a DocumentUpdate. + * + * @param docId the ID of the update + * @param docType the document type that this update is valid for + */ + public DocumentUpdate(DocumentType docType, DocumentId docId) { + this(docType, docId, new ArrayList<FieldUpdate>()); + } + + /** + * Creates a new document update using a reader + */ + public DocumentUpdate(DocumentUpdateReader reader) { + docId = null; + documentType = null; + fieldUpdates = new ArrayList<>(); + fieldPathUpdates = new ArrayList<>(); + reader.read(this); + } + + /** + * Creates a DocumentUpdate. + * + * @param docId the ID of the update + * @param docType the document type that this update is valid for + */ + public DocumentUpdate(DocumentType docType, String docId) { + this(docType, new DocumentId(docId)); + } + + private DocumentUpdate(DocumentType docType, DocumentId docId, List<FieldUpdate> fieldUpdates) { + this.docId = docId; + this.documentType = docType; + this.fieldUpdates = fieldUpdates; + this.fieldPathUpdates = new ArrayList<>(); + } + + public DocumentId getId() { + return docId; + } + + /** + * Sets the document id of the document to update. + * Use only while deserializing - changing the document id after creation has undefined behaviour. + */ + public void setId(DocumentId id) { + docId = id; + } + + /** + * Applies this document update. + * + * @param doc the document to apply the update to + * @return a reference to itself + * @throws IllegalArgumentException if the document does not have the same document type as this update + */ + public DocumentUpdate applyTo(Document doc) { + if (!documentType.equals(doc.getDataType())) { + throw new IllegalArgumentException( + "Document " + doc + " must have same type as update, which is type " + documentType); + } + + for (FieldUpdate fieldUpdate : fieldUpdates) { + fieldUpdate.applyTo(doc); + } + for (FieldPathUpdate fieldPathUpdate : fieldPathUpdates) { + fieldPathUpdate.applyTo(doc); + } + return this; + } + + /** + * Get an unmodifiable list of all field updates that this document update specifies. + * + * @return a list of all FieldUpdates in this DocumentUpdate + */ + public List<FieldUpdate> getFieldUpdates() { + return Collections.unmodifiableList(fieldUpdates); + } + + /** + * Get an unmodifiable list of all field path updates this document update specifies. + * + * @return Returns a list of all field path updates in this document update. + */ + public List<FieldPathUpdate> getFieldPathUpdates() { + return Collections.unmodifiableList(fieldPathUpdates); + } + + /** Returns the type of the document this updates + * + * @return The documentype of the document + */ + public DocumentType getDocumentType() { + return documentType; + } + + /** + * Sets the document type. Use only while deserializing - changing the document type after creation + * has undefined behaviour. + */ + public void setDocumentType(DocumentType type) { + documentType = type; + } + + /** + * Get the field update at the specified index in the list of field updates. + * + * @param index the index of the FieldUpdate to return + * @return the FieldUpdate at the specified index + * @throws IndexOutOfBoundsException if index is out of range + */ + public FieldUpdate getFieldUpdate(int index) { + return fieldUpdates.get(index); + } + + /** + * Replaces the field update at the specified index in the list of field updates. + * + * @param index index of the FieldUpdate to replace + * @param upd the FieldUpdate to be stored at the specified position + * @return the FieldUpdate previously at the specified position + * @throws IndexOutOfBoundsException if index is out of range + */ + public FieldUpdate setFieldUpdate(int index, FieldUpdate upd) { + return fieldUpdates.set(index, upd); + } + + /** + * Returns the update for a field + * + * @param field the field to return the update of + * @return the update for the field, or null if that field has no update in this + */ + public FieldUpdate getFieldUpdate(Field field) { + return getFieldUpdate(field.getName()); + } + + /** Removes all field updates from the list for field updates. */ + public void clearFieldUpdates() { + fieldUpdates.clear(); + } + + /** + * Returns the update for a field name + * + * @param fieldName the field name to return the update of + * @return the update for the field, or null if that field has no update in this + */ + public FieldUpdate getFieldUpdate(String fieldName) { + for (FieldUpdate fieldUpdate : fieldUpdates) { + if (fieldUpdate.getField().getName().equals(fieldName)) { + return fieldUpdate; + } + } + return null; + } + + /** + * Assigns the field updates of this document update. + * This document update receives ownership of the list - it can not be subsequently used + * by the caller. The list may not be unmodifiable. + * + * @param fieldUpdates the new list of updates of this + * @throws NullPointerException if the argument passed is null + */ + public void setFieldUpdates(List<FieldUpdate> fieldUpdates) { + if (fieldUpdates == null) { + throw new NullPointerException("The field updates of a document update can not be null"); + } + this.fieldUpdates = fieldUpdates; + } + + /** + * Get the number of field updates in this document update. + * + * @return the size of the List of FieldUpdates + */ + public int size() { + return fieldUpdates.size(); + } + + /** + * Adds the given {@link FieldUpdate} to this DocumentUpdate. If this DocumentUpdate already contains a FieldUpdate + * for the named field, the content of the given FieldUpdate is added to the existing one. + * + * @param update The FieldUpdate to add to this DocumentUpdate. + * @return This, to allow chaining. + * @throws IllegalArgumentException If the {@link DocumentType} of this DocumentUpdate does not have a corresponding + * field. + */ + public DocumentUpdate addFieldUpdate(FieldUpdate update) { + String fieldName = update.getField().getName(); + if (!documentType.hasField(fieldName)) { + throw new IllegalArgumentException("Document type '" + documentType.getName() + "' does not have field '" + + fieldName + "'."); + } + FieldUpdate prevUpdate = getFieldUpdate(fieldName); + if (prevUpdate != update) { + if (prevUpdate != null) { + prevUpdate.addAll(update); + } else { + fieldUpdates.add(update); + } + } + return this; + } + + /** + * Adds a field path update to perform on the document. + * + * @return a reference to itself. + */ + public DocumentUpdate addFieldPathUpdate(FieldPathUpdate fieldPathUpdate) { + fieldPathUpdates.add(fieldPathUpdate); + return this; + } + + // TODO: Remove this when we figure out correct behaviour. + + public void addFieldUpdateNoCheck(FieldUpdate fieldUpdate) { + fieldUpdates.add(fieldUpdate); + } + + /** + * Adds all the field- and field path updates of the given document update to this. If the given update refers to a + * different document or document type than this, this method throws an exception. + * + * @param update The update whose content to add to this. + * @throws IllegalArgumentException If the {@link DocumentId} or {@link DocumentType} of the given DocumentUpdate + * does not match the content of this. + */ + public void addAll(DocumentUpdate update) { + if (update == null) { + return; + } + if (!docId.equals(update.docId)) { + throw new IllegalArgumentException("Expected " + docId + ", got " + update.docId + "."); + } + if (!documentType.equals(update.documentType)) { + throw new IllegalArgumentException("Expected " + documentType + ", got " + update.documentType + "."); + } + for (FieldUpdate fieldUpd : update.fieldUpdates) { + addFieldUpdate(fieldUpd); + } + for (FieldPathUpdate pathUpd : update.fieldPathUpdates) { + addFieldPathUpdate(pathUpd); + } + } + + /** + * Removes the field update at the specified position in the list of field updates. + * + * @param index the index of the FieldUpdate to remove + * @return the FieldUpdate previously at the specified position + * @throws IndexOutOfBoundsException if index is out of range + */ + public FieldUpdate removeFieldUpdate(int index) { + return fieldUpdates.remove(index); + } + + /** + * Returns the document type of this document update. + * + * @return the document type of this document update + */ + public DocumentType getType() { + return documentType; + } + + public final void serialize(GrowableByteBuffer buf) { + serialize(DocumentSerializerFactory.create42(buf)); + } + + public void serialize(DocumentUpdateWriter data) { + data.write(this); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DocumentUpdate)) return false; + + DocumentUpdate that = (DocumentUpdate) o; + + if (docId != null ? !docId.equals(that.docId) : that.docId != null) return false; + if (documentType != null ? !documentType.equals(that.documentType) : that.documentType != null) return false; + if (fieldPathUpdates != null ? !fieldPathUpdates.equals(that.fieldPathUpdates) : that.fieldPathUpdates != null) + return false; + if (fieldUpdates != null ? !fieldUpdates.equals(that.fieldUpdates) : that.fieldUpdates != null) return false; + if (createIfNonExistent != that.createIfNonExistent) return false; + + return true; + } + + @Override + public int hashCode() { + int result = docId != null ? docId.hashCode() : 0; + result = 31 * result + (fieldUpdates != null ? fieldUpdates.hashCode() : 0); + result = 31 * result + (fieldPathUpdates != null ? fieldPathUpdates.hashCode() : 0); + result = 31 * result + (documentType != null ? documentType.hashCode() : 0); + return result; + } + + @Override + public String toString() { + StringBuilder string = new StringBuilder(); + string.append("update of document '"); + string.append(docId); + string.append("': "); + string.append("create-if-non-existent="); + string.append(createIfNonExistent ? "true" : "false"); + string.append(": "); + string.append("["); + + for (Iterator<FieldUpdate> i = fieldUpdates.iterator(); i.hasNext();) { + FieldUpdate fieldUpdate = i.next(); + string.append(fieldUpdate); + if (i.hasNext()) { + string.append(", "); + } + } + string.append("]"); + + if (fieldPathUpdates.size() > 0) { + string.append(" [ "); + for (FieldPathUpdate up : fieldPathUpdates) { + string.append(up.toString() + " "); + } + string.append(" ]"); + } + + return string.toString(); + } + + public Iterator<FieldPathUpdate> iterator() { + return fieldPathUpdates.iterator(); + } + + /** + * Returns whether or not this field update contains any field- or field path updates. + * + * @return True if this update is empty. + */ + public boolean isEmpty() { + return fieldUpdates.isEmpty() && fieldPathUpdates.isEmpty(); + } + + /** + * Sets whether this update should create the document it updates if that document does not exist. + * In this case an empty document is created before the update is applied. + * + * @since 5.17 + * @param value Whether the document it updates should be created. + */ + public void setCreateIfNonExistent(boolean value) { + createIfNonExistent = value; + } + + /** + * Gets whether this update should create the document it updates if that document does not exist. + * + * @since 5.17 + * @return Whether the document it updates should be created. + */ + public boolean getCreateIfNonExistent() { + return createIfNonExistent; + } +} diff --git a/document/src/main/java/com/yahoo/document/DocumentUtil.java b/document/src/main/java/com/yahoo/document/DocumentUtil.java new file mode 100644 index 00000000000..bce88a67ba8 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/DocumentUtil.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +/** + * Class containing static utility function related to documents. + * @author einarmr + * @since 5.1.9 + */ +public class DocumentUtil { + /** + * A convenience method that can be used to calculate a max pending queue size given + * the number of threads processing the documents, their size, and the memory available. + * + * @return the max pending size (in bytes) that should be used. + */ + public static int calculateMaxPendingSize(double maxConcurrentFactor, double documentExpansionFactor, int containerCoreMemoryMb) { + final long heapBytes = Runtime.getRuntime().maxMemory(); + final long heapMb = heapBytes / 1024L / 1024L; + final double maxPendingMb = ((double) (heapMb - containerCoreMemoryMb)) / (1.0d + (maxConcurrentFactor * documentExpansionFactor)); + long maxPendingBytes = ((long) (maxPendingMb * 1024.0d)) * 1024L; + if (maxPendingBytes < (1024L * 1024L)) { + maxPendingBytes = 1024L * 1024L; //1 MB + } + if (maxPendingBytes > (heapBytes / 5L)) { + maxPendingBytes = heapBytes / 5L; //we do not want a maxPendingBytes greater than 1/5 heap (we probably have a very low expansion factor) + } + if (maxPendingBytes > (1<<30)) { //we don't want a maxPendingBytes greater than 1G + maxPendingBytes = 1<<30; + } + return (int) maxPendingBytes; + } +} diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java new file mode 100644 index 00000000000..86543916b42 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/Field.java @@ -0,0 +1,259 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.collections.BobHash; +import com.yahoo.document.fieldset.DocIdOnly; +import com.yahoo.document.fieldset.FieldSet; +import com.yahoo.document.fieldset.NoFields; +import com.yahoo.vespa.objects.FieldBase; + +import java.io.Serializable; + +/** + * A name and type. Fields are contained in document types to describe their fields, + * but is also used to represent name/type pairs which are not part of document types. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author bratseth + */ +public class Field extends FieldBase implements FieldSet, Comparable, Serializable { + + protected DataType dataType; + protected int fieldId; + private int fieldIdV6; + private boolean isHeader; + private boolean forcedId = false; + + /** + * Creates a new field. + * + * @param name The name of the field + * @param dataType The datatype of the field + * @param isHeader Whether this is a "header" field or a "content" field + * (true = "header"). + */ + public Field(String name, int id, DataType dataType, boolean isHeader) { + super(name); + this.fieldId = id; + this.fieldIdV6 = id; + this.dataType = dataType; + this.isHeader = isHeader; + this.forcedId = true; + validateId(id, null, Document.SERIALIZED_VERSION); + } + + public Field(String name) { + this(name, DataType.NONE); + } + + + /** + * Creates a new field. + * + * @param name The name of the field + * @param dataType The datatype of the field + * @param isHeader Whether this is a "header" field or a "content" field + * (true = "header"). + * @param owner the owning document (used to check for id collisions) + */ + public Field(String name, DataType dataType, boolean isHeader, DocumentType owner) { + this(name, 0, dataType, isHeader); + this.fieldId = calculateIdV7(owner); + this.fieldIdV6 = calculateIdV6(owner); + this.forcedId = false; + } + + /** + * Creates a new field. + * + * @param name The name of the field + * @param dataType The datatype of the field + * @param isHeader Whether this is a "header" field or a "content" field + * (true = "header"). + */ + public Field(String name, DataType dataType, boolean isHeader) { + this(name, dataType, isHeader, null); + } + + /** + * Constructor for <b>header</b> fields + * + * @param name The name of the field + * @param dataType The datatype of the field + */ + public Field(String name, DataType dataType) { + this(name, dataType, true); + } + + /** + * Creates a field with a new name and the other properties + * (excluding the id and owner) copied from another field + */ + // TODO: Decide on one copy/clone idiom and do it for this and all it is calling + public Field(String name, Field field) { + this(name, field.dataType, field.isHeader, null); + } + + /** + * The field id must be unique within a document type, and also + * within a (unknown at this time) hierarchy of document types. + * In addition it should be as resilient to doctype content changes + * and inheritance hierarchy changes as possible. + * All of this is enforced for names, so id's should follow names. + * Therefore we hash on name. + */ + private int calculateIdV6(DocumentType owner) { + int newId = BobHash.hash(getName()); // Using a portfriendly hash + if (newId < 0) newId = -newId; // Highest bit is reserved to tell 7-bit id's from 31-bit ones + validateId(newId, owner, 6); + return newId; + } + + public int compareTo(Object o) { + return fieldId - ((Field) o).fieldId; + } + + /** + * The field id must be unique within a document type, and also + * within a (unknown at this time) hierarchy of document types. + * In addition it should be as resilient to doctype content changes + * and inheritance hierarchy changes as possible. + * All of this is enforced for names, so id's should follow names. + * Therefore we hash on name. + */ + protected int calculateIdV7(DocumentType owner) { + String combined = getName() + dataType.getId(); + + int newId = BobHash.hash(combined); // Using a portfriendly hash + if (newId < 0) newId = -newId; // Highest bit is reserved to tell 7-bit id's from 31-bit ones + validateId(newId, owner, Document.SERIALIZED_VERSION); + return newId; + } + + /** + * Sets the id of this field. Don't do this unless you know what you are doing + * + * @param newId the id - if this is less than 100 it will cause document to serialize + * using just one byte for this field id. 100-127 are reserved values + * @param owner the owning document, this is checked for collisions and notified + * of the id change. It can not be null + */ + public void setId(int newId, DocumentType owner) { + if (owner == null) { + throw new NullPointerException("Can not assign an id of " + this + " without knowing the owner"); + } + + validateId(newId, owner, Document.SERIALIZED_VERSION); + + owner.removeField(getName()); + this.fieldId = newId; + this.fieldIdV6 = newId; + this.forcedId = true; + owner.addField(this); + } + + private void validateId(int newId, DocumentType owner, int version) { + if (newId >= 100 && newId <= 127) { + throw new IllegalArgumentException("Attempt to set the id of " + this + " to " + newId + + " failed, values from 100 to 127 " + "are reserved for internal use"); + } + + if ((newId & 0x80000000) != 0) // Highest bit must not be set + { + throw new IllegalArgumentException("Attempt to set the id of " + this + " to " + newId + + " failed, negative id values " + " are illegal"); + } + + + if (owner == null) return; + { + Field existing = owner.getField(newId, version); + if (existing != null && !existing.getName().equals(getName())) { + throw new IllegalArgumentException("Couldn't set id of " + this + " to " + newId + ", " + existing + + " already has this id in " + owner); + } + } + } + + /** @return Returns the datatype of the field */ + public final DataType getDataType() { + return dataType; + } + + /** + * Set the data type of the field. This will cause recalculation of fieldid for version 7+. + * + * @deprecated do not use + * @param type The new type of the field. + */ + @Deprecated // Do not remove on Vespa 6 + public void setDataType(DataType type) { + dataType = type; + fieldId = calculateIdV7(null); + forcedId = false; + } + + /** Returns the numeric ID used to represent this field when serialized */ + public final int getId(int version) { + return (version > 6) ? getId() : getIdV6(); + } + + public final int getId() { + return fieldId; + } + + public final int getIdV6() { + return fieldIdV6; + } + + /** + * + * @return true if the field has a forced id + */ + public final boolean hasForcedId() { + return forcedId; + } + + /** @return Returns true if this field should be a part of "header" serializations. */ + public boolean isHeader() { + return isHeader; + } + + /** Sets whether this is a header field */ + public void setHeader(boolean header) { + this.isHeader = header; + } + + /** Two fields are equal if they have the same name and the same data type */ + @Override + public boolean equals(Object o) { + return this == o || o instanceof Field && super.equals(o) && dataType.equals(((Field) o).dataType); + } + + @Override + public int hashCode() { + return getId(); + } + + public String toString() { + return super.toString() + "(" + dataType + ")"; + } + + @Override + public boolean contains(FieldSet o) { + if (o instanceof NoFields || o instanceof DocIdOnly) { + return true; + } + + if (o instanceof Field) { + return equals(o); + } + + return false; + } + + @Override + public FieldSet clone() throws CloneNotSupportedException { + return (Field)super.clone(); + } +} diff --git a/document/src/main/java/com/yahoo/document/FieldPath.java b/document/src/main/java/com/yahoo/document/FieldPath.java new file mode 100755 index 00000000000..0497f2b139d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/FieldPath.java @@ -0,0 +1,127 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * This class represents a path into a document, that can be used to iterate through the document and extract the field + * values you're interested in. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class FieldPath implements Iterable<FieldPathEntry> { + + private final List<FieldPathEntry> list; + /** + * Constructs an empty path. + */ + public FieldPath() { + list = Collections.emptyList(); + } + + /** + * Constructs a path containing the entries of the specified path, in the order they are returned by that path's + * iterator. + * + * @param path The path whose entries are to be placed into this path. + * @throws NullPointerException If the specified path is null. + */ + public FieldPath(FieldPath path) { + this(path.list); + } + + public FieldPath(List<FieldPathEntry> path) { + list = Collections.unmodifiableList(path); + } + + public int size() { return list.size(); } + public FieldPathEntry get(int index) { return list.get(index); } + public boolean isEmpty() { return list.isEmpty(); } + public Iterator<FieldPathEntry> iterator() { return list.iterator(); } + public List<FieldPathEntry> getList() { return list; } + + /** + * Compares this field path with the given field path, returns true if the field path starts with the other. + * + * @param other The field path to compare with. + * @return Returns true if this field path starts with the other field path, otherwise false + */ + public boolean startsWith(FieldPath other) { + if (other.size() > size()) { + return false; + } + + for (int i = 0; i < other.size(); i++) { + if (!other.get(i).equals(get(i))) { + return false; + } + } + + return true; + } + + /** + * @return Returns the datatype we can expect this field path to return. + */ + public DataType getResultingDataType() { + if (isEmpty()) { + return null; + } + + return get(size() - 1).getResultingDataType(); + } + + /** + * Convenience method to build a field path from a path string. This is a simple proxy for {@link + * DataType#buildFieldPath(String)}. + * + * @param fieldType The data type of the value to build a path for. + * @param fieldPath The path string to parse. + * @return The corresponding field path object. + */ + public static FieldPath newInstance(DataType fieldType, String fieldPath) { + return fieldType.buildFieldPath(fieldPath); + } + + @Override + public String toString() { + StringBuilder out = new StringBuilder(); + DataType prevType = null; + for (FieldPathEntry entry : this) { + FieldPathEntry.Type type = entry.getType(); + switch (type) { + case STRUCT_FIELD: + if (out.length() > 0) { + out.append("."); + } + Field field = entry.getFieldRef(); + out.append(field.getName()); + prevType = field.getDataType(); + break; + case ARRAY_INDEX: + out.append("[").append(entry.getLookupIndex()).append("]"); + break; + case MAP_KEY: + break; + case MAP_ALL_KEYS: + out.append(".key"); + break; + case MAP_ALL_VALUES: + out.append(".value"); + break; + case VARIABLE: + if (prevType instanceof ArrayDataType) { + out.append("[$").append(entry.getVariableName()).append("]"); + } else if (prevType instanceof WeightedSetDataType || prevType instanceof MapDataType) { + out.append("{$").append(entry.getVariableName()).append("}"); + } else { + out.append("$").append(entry.getVariableName()); + } + } + } + return out.toString(); + } +} diff --git a/document/src/main/java/com/yahoo/document/FieldPathEntry.java b/document/src/main/java/com/yahoo/document/FieldPathEntry.java new file mode 100755 index 00000000000..d542acd430e --- /dev/null +++ b/document/src/main/java/com/yahoo/document/FieldPathEntry.java @@ -0,0 +1,312 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; + +/** + * @author thomasg + */ +public class FieldPathEntry { + public enum Type { + STRUCT_FIELD, + ARRAY_INDEX, + MAP_KEY, + MAP_ALL_KEYS, + MAP_ALL_VALUES, + VARIABLE + } + + private final Type type; + private final int lookupIndex; + private final FieldValue lookupKey; + private final String variableName; + private final Field fieldRef; + private final DataType resultingDataType; + + public DataType getResultingDataType() { + return resultingDataType; + } + + public Type getType() { + return type; + } + + public Field getFieldRef() { + return fieldRef; + } + + public int getLookupIndex() { + return lookupIndex; + } + + public FieldValue getLookupKey() { + return lookupKey; + } + + public String getVariableName() { + return variableName; + } + + public String toString() { + String retVal = type.toString() + ": "; + switch (type) { + case STRUCT_FIELD: + retVal += getFieldRef(); + break; + case ARRAY_INDEX: + retVal += getLookupIndex(); + break; + case MAP_KEY: + retVal += getLookupKey(); + break; + case MAP_ALL_KEYS: + case MAP_ALL_VALUES: + break; + case VARIABLE: + retVal += getVariableName(); + break; + } + return retVal; + } + + /** + * Creates a new field path entry that references a struct field. + * For these kinds of field path entries, getFieldRef() is valid. + * + * @param fieldRef The field to look up in the struct. + * @return The new field path entry + */ + public static FieldPathEntry newStructFieldEntry(Field fieldRef) { + return new FieldPathEntry(fieldRef); + } + + /** + * Creates a new field path entry that references an array index. + * + * @param lookupIndex The index to look up + * @param resultingDataType The datatype of the contents of the array + * @return The new field path entry + */ + public static FieldPathEntry newArrayLookupEntry(int lookupIndex, DataType resultingDataType) { + return new FieldPathEntry(lookupIndex, resultingDataType); + } + + /** + * Creates a new field path entry that references a map or weighted set. + * + * @param lookupKey The value of the key in the map or weighted set to recurse into. + * @param resultingDataType The datatype of values in the map or weighted set. + * @return The new field path entry + */ + public static FieldPathEntry newMapLookupEntry(FieldValue lookupKey, DataType resultingDataType) { + return new FieldPathEntry(lookupKey, resultingDataType); + } + + /** + * Creates a new field path entry that digs through all the keys of a map or weighted set. + * + * @param resultingDataType The datatype of the keys in the map or weighted set. + * @return The new field path entry. + */ + public static FieldPathEntry newAllKeysLookupEntry(DataType resultingDataType) { + return new FieldPathEntry(true, false, resultingDataType); + } + + /** + * Creates a new field path entry that digs through all the values of a map or weighted set. + * + * @param resultingDataType The datatype of the values in the map or weighted set. + * @return The new field path entry. + */ + public static FieldPathEntry newAllValuesLookupEntry(DataType resultingDataType) { + return new FieldPathEntry(false, true, resultingDataType); + } + + /** + * Creates a new field path entry that digs through all the keys in a map or weighted set, or all the indexes of an array, + * an sets the given variable name as it does so (or, if the variable is set, uses the set variable to look up the + * collection. + * + * @param variableName The name of the variable to lookup in the collection + * @param resultingDataType The value type of the collection we're digging through + * @return The new field path entry. + */ + public static FieldPathEntry newVariableLookupEntry(String variableName, DataType resultingDataType) { + return new FieldPathEntry(variableName, resultingDataType); + } + + private FieldPathEntry(Field fieldRef) { + type = Type.STRUCT_FIELD; + lookupIndex = 0; + lookupKey = null; + variableName = null; + this.fieldRef = fieldRef; + resultingDataType = fieldRef.getDataType(); + } + + private FieldPathEntry(int lookupIndex, DataType resultingDataType) { + type = Type.ARRAY_INDEX; + this.lookupIndex = lookupIndex; + lookupKey = null; + variableName = null; + fieldRef = null; + this.resultingDataType = resultingDataType; + } + + private FieldPathEntry(FieldValue lookupKey, DataType resultingDataType) { + type = Type.MAP_KEY; + lookupIndex = 0; + this.lookupKey = lookupKey; + variableName = null; + fieldRef = null; + this.resultingDataType = resultingDataType; + } + + private FieldPathEntry(boolean keysOnly, boolean valuesOnly, DataType resultingDataType) { + type = keysOnly ? Type.MAP_ALL_KEYS : Type.MAP_ALL_VALUES; + lookupIndex = 0; + lookupKey = null; + variableName = null; + fieldRef = null; + this.resultingDataType = resultingDataType; + } + + private FieldPathEntry(String variableName, DataType resultingDataType) { + type = Type.VARIABLE; + lookupIndex = 0; + lookupKey = null; + this.variableName = variableName; + fieldRef = null; + this.resultingDataType = resultingDataType; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FieldPathEntry that = (FieldPathEntry) o; + + if (lookupIndex != that.lookupIndex) return false; + if (fieldRef != null ? !fieldRef.equals(that.fieldRef) : that.fieldRef != null) return false; + if (lookupKey != null ? !lookupKey.equals(that.lookupKey) : that.lookupKey != null) return false; + if (resultingDataType != null ? !resultingDataType.equals(that.resultingDataType) : that.resultingDataType != null) + return false; + if (type != that.type) return false; + if (variableName != null ? !variableName.equals(that.variableName) : that.variableName != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = type.hashCode(); + result = 31 * result + lookupIndex; + result = 31 * result + (lookupKey != null ? lookupKey.hashCode() : 0); + result = 31 * result + (variableName != null ? variableName.hashCode() : 0); + result = 31 * result + (fieldRef != null ? fieldRef.hashCode() : 0); + result = 31 * result + (resultingDataType != null ? resultingDataType.hashCode() : 0); + return result; + } + + public static class KeyParseResult { + public String parsed; + public int consumedChars; + + public KeyParseResult(String parsed, int consumedChars) { + this.parsed = parsed; + this.consumedChars = consumedChars; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + KeyParseResult that = (KeyParseResult) o; + + if (consumedChars != that.consumedChars) return false; + if (!parsed.equals(that.parsed)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = parsed.hashCode(); + result = 31 * result + consumedChars; + return result; + } + + @Override + public String toString() { + return "KeyParseResult(parsed=\"" + parsed + "\", consumedChars=" + consumedChars + ")"; + } + } + + private static int parseQuotedString(String key, int offset, + int len, StringBuilder builder) + { + for (; offset < len && key.charAt(offset) != '"'; ++offset) { + if (key.charAt(offset) == '\\') { + ++offset; // Skip escape backslash + if (offset == len || key.charAt(offset) != '"') { + throw new IllegalArgumentException("Escaped key '" + key + "' has bad quote character escape sequence. Expected '\"'"); + } + } + if (offset < len) { + builder.append(key.charAt(offset)); + } + } + if (offset < len && key.charAt(offset) == '"') { + return offset + 1; + } else { + throw new IllegalArgumentException("Escaped key '" + key + "' is incomplete. No matching '\"'"); + } + } + + private static int skipWhitespace(String str, int offset, int len) { + while (offset < len && Character.isSpaceChar(str.charAt(offset))) { + ++offset; + } + return offset; + } + + /** + * Parse a field path map key of the form {xyz} or {"xyz"} with optional trailing data. + * If the key contains a '}' or '"' character, the key must be in quotes and all + * double-quote characters must be escaped. Only '"' chars may be escaped. Any + * trailing string data past the '}' is ignored. + * + * @param key Part of a field path that contains a key at its start + * @return A parse result containing the parsed/unescaped key and the number + * of input characters the parse consumed. Does not include any characters + * beyond the '}' char. + */ + public static KeyParseResult parseKey(String key) { + StringBuilder parsed = new StringBuilder(key.length()); + // Hooray for ad-hoc parsing + int len = key.length(); + int i = 0; + if (i < len && key.charAt(0) == '{') { + i = skipWhitespace(key, i + 1, len); + if (i < len && key.charAt(i) == '"') { + i = parseQuotedString(key, i + 1, len, parsed); + } else { + // No quoting, use all of string until '}' verbatim + while (i < len && key.charAt(i) != '}') { + parsed.append(key.charAt(i)); + ++i; + } + } + i = skipWhitespace(key, i, len); + if (i < len && key.charAt(i) == '}') { + return new KeyParseResult(parsed.toString(), i + 1); + } else { + throw new IllegalArgumentException("Key '" + key + "' is incomplete. No matching '}'"); + } + } else { + throw new IllegalArgumentException("Key '" + key + "' does not start with '{'"); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/Generated.java b/document/src/main/java/com/yahoo/document/Generated.java new file mode 100644 index 00000000000..40026eed51f --- /dev/null +++ b/document/src/main/java/com/yahoo/document/Generated.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Classes generated by vespa-documentgen-plugin are annotated with this. It + * differs from <code>javax.annotation.Generated</code> in that the retention + * policy is Runtime. + * + * @author <a href="mailto:vegardh@yahoo-inc.com">Vegard Havdal</a> + */ +@Retention(RetentionPolicy.RUNTIME) +public @interface Generated { + +} diff --git a/document/src/main/java/com/yahoo/document/GlobalId.java b/document/src/main/java/com/yahoo/document/GlobalId.java new file mode 100644 index 00000000000..d1ab8c9cea9 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/GlobalId.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.collections.MD5; +import com.yahoo.document.idstring.IdString; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.objects.Deserializer; +import com.yahoo.vespa.objects.Serializer; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; + +/** + * Implements an incredibly light-weight version of the document global id. There is a lot of functionality in the C++ + * version of this that is missing. However, this should be sufficient for now. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GlobalId implements Comparable { + + /** + * The number of bytes in a global id. This must match the C++ constant in "document/base/globalid.h". + */ + public static final int LENGTH = 12; + + // The raw bytes that constitutes this global id. + private final byte[] raw; + + /** + * Constructs a new global id by copying the content of the given raw byte array. + * + * @param raw The array to copy. + */ + public GlobalId(byte[] raw) { + this.raw = new byte [12]; + int len = Math.min(LENGTH, raw.length); + System.arraycopy(raw, 0, this.raw, 0, len); + } + + /** + * Constructs a new global id from a document id string. + * + * @param id The document id to derive from. + */ + public GlobalId(IdString id) { + byte [] raw = MD5.md5.get().digest(id.toUtf8().wrap().array()); + long location = id.getLocation(); + this.raw = new byte [LENGTH]; + for (int i = 0; i < 4; ++i) { + this.raw[i] = (byte)((location >> (8 * i)) & 0xFF); + } + for (int i=4; i < LENGTH; i++) { + this.raw[i] = raw[i]; + } + } + + /** + * Constructs a global id by deserializing content from the given byte buffer. + * + * @param buf The buffer to deserialize from. + */ + public GlobalId(Deserializer buf) { + raw = buf.getBytes(null, LENGTH); + } + + /** + * Serializes the content of this global id into the given byte buffer. + * + * @param buf The buffer to serialize to. + */ + public void serialize(Serializer buf) { + buf.put(null, raw); + } + + /** + * Returns the raw byte array that constitutes this global id. + * + * @return The byte array. + */ + public byte[] getRawId() { + return raw; + } + + // Inherit doc from Object. + @Override + public int hashCode() { + return Arrays.hashCode(raw); + } + + public BucketId toBucketId() { + /** + * Explanation time: since Java was designed so mankind could suffer, + * shift ops on bytes have an implicit int conversion with sign-extend. + * When a byte is negative, you end up with an int/long with a 0xFFFFFF + * prefix, in turn causing your other friendly bitwise ORs to act + * pretty far from what was originally intended. + * To get around this, we explicitly sign extend before the compiler can + * do so for us and make sure to OR away any sign extensions. + */ + long location = ((long)raw[0] & 0xFF) + | (((long)raw[1] & 0xFF) << 8) + | (((long)raw[2] & 0xFF) << 16) + | (((long)raw[3] & 0xFF) << 24); + long md5 = 0; + for (int i = 4, j = 0; i < LENGTH; i++, j += 8) { + md5 |= ((long)raw[i] & 0xFF) << j; + } + // Drumroll: this is why 'location' is of type long. Otherwise, the + // ORing would sign-extend it and cause havoc when its MSB is set. + long rawBucketId = (md5 & 0xFFFFFFFF00000000L) | location; + return new BucketId(58, rawBucketId); + } + + // Inherit doc from Object. + @Override + public boolean equals(Object obj) { + if (!(obj instanceof GlobalId)) { + return false; + } + GlobalId rhs = (GlobalId) obj; + return Arrays.equals(raw, rhs.raw); + } + + public int compareTo(Object o) { + GlobalId other = (GlobalId) o; + + for (int i=0 ; i<LENGTH; i++) { + int thisByte = 0xF & (int) raw[i]; + int otherByte = 0xF & (int) other.raw[i]; + + if (thisByte < otherByte) { + return -1; + } else if (thisByte > otherByte) { + return 1; + } + } + return 0; + } + + @Override + public String toString() { + StringBuilder strb = new StringBuilder(50); + for (byte b : raw) { + strb.append(" ").append(0xFF & (int) b); + } + return strb.toString().trim(); + } + +} diff --git a/document/src/main/java/com/yahoo/document/MapDataType.java b/document/src/main/java/com/yahoo/document/MapDataType.java new file mode 100644 index 00000000000..2f3c6be99dc --- /dev/null +++ b/document/src/main/java/com/yahoo/document/MapDataType.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.MapFieldValue; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents a map type. + * + * @author vegardh + */ +public class MapDataType extends DataType { + + private DataType keyType; + private DataType valueType; + + public MapDataType(DataType keyType, DataType valueType, int id) { + super("Map<"+keyType.getName()+","+valueType.getName()+">", id); + this.keyType=keyType; + this.valueType = valueType; + + } + + public MapDataType(DataType keyType, DataType valueType) { + this(keyType, valueType, 0); + setId(getName().toLowerCase().hashCode()); + } + + @Override + public MapDataType clone() { + MapDataType type = (MapDataType)super.clone(); + type.keyType = keyType.clone(); + type.valueType = valueType.clone(); + return type; + } + + @Override + protected FieldValue createByReflection(Object arg) { return null; } + + @Override + public boolean isValueCompatible(FieldValue value) { + return value.getDataType().equals(this); + } + + public DataType getKeyType() { + return keyType; + } + + public DataType getValueType() { + return valueType; + } + + /** + * Sets the key type of this MapDataType. WARNING! Do not use! Only to be used by config system! + */ + public void setKeyType(DataType keyType) { + this.keyType = keyType; + } + + /** + * Sets the key type of this MapDataType. WARNING! Do not use! Only to be used by config system! + */ + public void setValueType(DataType valueType) { + this.valueType = valueType; + } + + + @Override + public MapFieldValue createFieldValue() { + return new MapFieldValue(this); + } + + @Override + public Class getValueClass() { + return MapFieldValue.class; + } + + @Override + protected void register(DocumentTypeManager manager, + List<DataType> seenTypes) { + seenTypes.add(this); + if (!seenTypes.contains(getKeyType())) { + getKeyType().register(manager, seenTypes); + } + if (!seenTypes.contains(getValueType())) { + getValueType().register(manager, seenTypes); + } + super.register(manager, seenTypes); + } + + public static FieldPath buildFieldPath(String remainFieldName, DataType keyType, DataType valueType) { + if (remainFieldName.length() > 0 && remainFieldName.charAt(0) == '{') { + FieldPathEntry.KeyParseResult result = FieldPathEntry.parseKey(remainFieldName); + String keyValue = result.parsed; + + FieldPath path = valueType.buildFieldPath(skipDotInString(remainFieldName, result.consumedChars - 1)); + List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(path.getList()); + + if (remainFieldName.charAt(1) == '$') { + tmpPath.add(0, FieldPathEntry.newVariableLookupEntry(keyValue.substring(1), valueType)); + } else { + FieldValue fv = keyType.createFieldValue(); + fv.assign(keyValue); + tmpPath.add(0, FieldPathEntry.newMapLookupEntry(fv, valueType)); + } + + return new FieldPath(tmpPath); + + } else if (remainFieldName.startsWith("key")) { + FieldPath path = keyType.buildFieldPath(skipDotInString(remainFieldName, 2)); + List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(path.getList()); + tmpPath.add(0, FieldPathEntry.newAllKeysLookupEntry(keyType)); + return new FieldPath(tmpPath); + } else if (remainFieldName.startsWith("value")) { + FieldPath path = valueType.buildFieldPath(skipDotInString(remainFieldName, 4)); + List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(path.getList()); + tmpPath.add(0, FieldPathEntry.newAllValuesLookupEntry(valueType)); + return new FieldPath(tmpPath); + } + + return keyType.buildFieldPath(remainFieldName); + } + + @Override + public FieldPath buildFieldPath(String remainFieldName) { + return buildFieldPath(remainFieldName, getKeyType(), getValueType()); + } + + @Override + public boolean isMultivalue() { return true; } + +} diff --git a/document/src/main/java/com/yahoo/document/NumericDataType.java b/document/src/main/java/com/yahoo/document/NumericDataType.java new file mode 100644 index 00000000000..20e02914a07 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/NumericDataType.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.vespa.objects.Ids; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class NumericDataType extends PrimitiveDataType { + // The global class identifier shared with C++. + public static int classId = registerClass(Ids.document + 52, NumericDataType.class); + /** + * Creates a datatype + * + * @param name the name of the type + * @param code the code (id) of the type + * @param type the field value used for this type + */ + protected NumericDataType(java.lang.String name, int code, Class type, Factory factory) { + super(name, code, type, factory); + } + + @Override + public NumericDataType clone() { + return (NumericDataType) super.clone(); + } +} diff --git a/document/src/main/java/com/yahoo/document/PositionDataType.java b/document/src/main/java/com/yahoo/document/PositionDataType.java new file mode 100644 index 00000000000..a2a1c6012a1 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/PositionDataType.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.geo.DegreesParser; +import com.yahoo.document.serialization.XmlStream; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public final class PositionDataType { + + public static final StructDataType INSTANCE = newInstance(); + public static final String STRUCT_NAME = "position"; + + public static final String FIELD_X = "x"; + public static final String FIELD_Y = "y"; + private static final Field FFIELD_X = INSTANCE.getField(FIELD_X); + private static final Field FFIELD_Y = INSTANCE.getField(FIELD_Y); + + private PositionDataType() { + // unreachable + } + + public static String renderAsString(Struct pos) { + StringBuilder buf = new StringBuilder(); + double ns = getYValue(pos).getInteger() / 1.0e6; + double ew = getXValue(pos).getInteger() / 1.0e6; + buf.append(ns < 0 ? "S" : "N"); + buf.append(ns < 0 ? (-ns) : ns); + buf.append(";"); + buf.append(ew < 0 ? "W" : "E"); + buf.append(ew < 0 ? (-ew) : ew); + return buf.toString(); + } + + public static void renderXml(Struct pos, XmlStream target) { + target.addContent(renderAsString(pos)); + } + + public static Struct valueOf(Integer x, Integer y) { + Struct ret = new Struct(INSTANCE); + ret.setFieldValue(FIELD_X, x != null ? new IntegerFieldValue(x) : null); + ret.setFieldValue(FIELD_Y, y != null ? new IntegerFieldValue(y) : null); + return ret; + } + + public static Struct fromLong(long val) { + return valueOf((int)(val >> 32), (int)val); + } + + public static Struct fromString(String str) { + try { + DegreesParser d = new DegreesParser(str); + return valueOf((int)(d.longitude * 1000000), (int)(d.latitude * 1000000)); + } catch (IllegalArgumentException e) { + // empty + } + String[] arr = str.split(";", 2); + return valueOf(Integer.parseInt(arr[0]), Integer.parseInt(arr[1])); + } + + public static IntegerFieldValue getXValue(FieldValue pos) { + return Struct.getFieldValue(pos, INSTANCE, FFIELD_X, IntegerFieldValue.class); + } + + public static IntegerFieldValue getYValue(FieldValue pos) { + return Struct.getFieldValue(pos, INSTANCE, FFIELD_Y, IntegerFieldValue.class); + } + + public static String getZCurveFieldName(String fieldName) { + return fieldName + "_zcurve"; + } + + public static String getPositionSummaryFieldName(String fieldName) { + // TODO for 6.0, rename to _position to use a field name that is actually legal + return fieldName + ".position"; + } + + public static String getDistanceSummaryFieldName(String fieldName) { + // TODO for 6.0, rename to _distance to use a field name that is actually legal + return fieldName + ".distance"; + } + + private static StructDataType newInstance() { + StructDataType ret = new StructDataType(STRUCT_NAME); + ret.addField(new Field(FIELD_X, DataType.INT)); + ret.addField(new Field(FIELD_Y, DataType.INT)); + return ret; + } +} diff --git a/document/src/main/java/com/yahoo/document/PrimitiveDataType.java b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java new file mode 100644 index 00000000000..a0024bb0497 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/PrimitiveDataType.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.objects.Ids; +import com.yahoo.vespa.objects.ObjectVisitor; + +import java.util.Objects; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class PrimitiveDataType extends DataType { + public static abstract class Factory { + public abstract FieldValue create(); + } + + // The global class identifier shared with C++. + public static final int classId = registerClass(Ids.document + 51, PrimitiveDataType.class); + private final Class<? extends FieldValue> valueClass; + private final Factory factory; + + /** + * Creates a datatype + * + * @param name the name of the type + * @param code the code (id) of the type + * @param factory the factory for creating field values of this type + */ + protected PrimitiveDataType(java.lang.String name, int code, Class<? extends FieldValue> valueClass, Factory factory) { + super(name, code); + Objects.requireNonNull(valueClass, "valueClass"); + Objects.requireNonNull(factory, "factory"); + this.valueClass = valueClass; + this.factory = factory; + } + + @Override + public PrimitiveDataType clone() { + return (PrimitiveDataType)super.clone(); + } + + public FieldValue createFieldValue() { + return factory.create(); + } + + @Override + public Class<? extends FieldValue> getValueClass() { + return valueClass; + } + + @Override + public boolean isValueCompatible(FieldValue value) { + return value != null && valueClass.isAssignableFrom(value.getClass()); + } + + @Override + public PrimitiveDataType getPrimitiveType() { + return this; + } + + @Override + public void visitMembers(ObjectVisitor visitor) { + super.visitMembers(visitor); + visitor.visit("valueclass", valueClass.getName()); + } +} diff --git a/document/src/main/java/com/yahoo/document/SimpleDocument.java b/document/src/main/java/com/yahoo/document/SimpleDocument.java new file mode 100644 index 00000000000..b461356ecef --- /dev/null +++ b/document/src/main/java/com/yahoo/document/SimpleDocument.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SimpleDocument { + + private final Document document; + + public SimpleDocument(Document document) { + this.document = document; + } + + public final Object get(Field field) { + return get(document, field); + } + + public final Object get(String fieldName) { + return get(document, document.getField(fieldName)); + } + + public final Object set(Field field, Object value) { + return set(document, field, value); + } + + public final Object set(String fieldName, Object value) { + return set(document.getField(fieldName), value); + } + + public final Object remove(Field field) { + return remove(document, field); + } + + public final Object remove(String fieldName) { + return remove(document.getField(fieldName)); + } + + public static Object get(StructuredFieldValue struct, Field field) { + return field == null ? null : unwrapValue(struct.getFieldValue(field)); + } + + public static Object set(StructuredFieldValue struct, Field field, Object value) { + return unwrapValue(struct.setFieldValue(field, wrapValue(field.getDataType(), value))); + } + + public static Object remove(StructuredFieldValue struct, Field field) { + return field == null ? null : unwrapValue(struct.removeFieldValue(field)); + } + + private static FieldValue wrapValue(DataType type, Object val) { + if (val == null) { + return null; + } + if (val instanceof FieldValue) { + return (FieldValue)val; + } + FieldValue ret = type.createFieldValue(); + ret.assign(val); + return ret; + } + + private static Object unwrapValue(FieldValue val) { + if (val == null) { + return null; + } + return val.getWrappedValue(); + } +} diff --git a/document/src/main/java/com/yahoo/document/StructDataType.java b/document/src/main/java/com/yahoo/document/StructDataType.java new file mode 100644 index 00000000000..3f26cbe054f --- /dev/null +++ b/document/src/main/java/com/yahoo/document/StructDataType.java @@ -0,0 +1,191 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.google.common.collect.ImmutableList; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.vespa.objects.Ids; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class StructDataType extends BaseStructDataType { + + public static final int classId = registerClass(Ids.document + 57, StructDataType.class); + private StructDataType superType = null; + + public StructDataType(String name) { + super(name); + } + + public StructDataType(int id,String name) { + super(id, name); + } + + @Override + public Struct createFieldValue() { + return new Struct(this); + } + + @Override + public FieldValue createFieldValue(Object o) { + Struct struct; + if (o.getClass().equals(Struct.class)) { + struct = new Struct(this); + } else { + // This indicates for example that o is a generated struct subtype, try the empty constructor + try { + struct = (Struct) o.getClass().getConstructor().newInstance(); + } catch (Exception e) { + // Fallback, let assign handle the error if o is completely bogus + struct = new Struct(this); + } + } + struct.assign(o); + return struct; + } + + @Override + public StructDataType clone() { + StructDataType type = (StructDataType) super.clone(); + type.superType = this.superType; + return type; + } + + public void assign(StructDataType type) { + super.assign(type); + superType = type.superType; + } + + @Override + public Field getField(Integer fieldId, int version) { + Field f = super.getField(fieldId, version); + if (f == null && superType != null) { + f = superType.getField(fieldId, version); + } + return f; + } + + @Override + public Field getField(String fieldName) { + Field f = super.getField(fieldName); + if (f == null && superType != null) { + f = superType.getField(fieldName); + } + return f; + } + + @Override + public Field getField(int id) { + Field f = super.getField(id); + if (f == null && superType != null) { + f = superType.getField(id); + } + return f; + } + + @Override + public void addField(Field field) { + if (hasField(field)) { + throw new IllegalArgumentException("Struct already has field " + field); + } + if ((superType != null) && superType.hasField(field)) { + throw new IllegalArgumentException(field.toString() + " already present in inherited type '" + superType.toString() + "', " + this.toString() + " cannot override."); + } + super.addField(field); + } + + @Override + public boolean hasField(Field field, int version) { + boolean f = super.hasField(field, version); + if (!f && superType != null) { + f = superType.hasField(field, version); + } + return f; + } + + @Override + public Collection<Field> getFields() { + if (superType == null) { + return Collections.unmodifiableCollection(super.getFields()); + } + Collection<Field> fieldsBuilder = new ArrayList<>(); + fieldsBuilder.addAll(super.getFields()); + fieldsBuilder.addAll(superType.getFields()); + return ImmutableList.copyOf(fieldsBuilder); + } + + public Collection<Field> getFieldsThisTypeOnly() { + return Collections.unmodifiableCollection(super.getFields()); + } + + @Override + public int getFieldCount() { + return getFields().size(); + } + + @Override + public Class getValueClass() { + return Struct.class; + } + + @Override + public boolean isValueCompatible(FieldValue value) { + if (!(value instanceof Struct)) { + return false; + } + Struct structValue = (Struct) value; + if (structValue.getDataType().inherits(this)) { + //the value is of this type; or the supertype of the value is of this type, etc.... + return true; + } + return false; + } + + public void inherit(StructDataType type) { + if (superType != null) { + throw new IllegalArgumentException("Already inherits type " + superType + ", multiple inheritance not currently supported."); + } + for (Field f : type.getFields()) { + if (hasField(f)) { + throw new IllegalArgumentException(f + " already present in " + type + ", " + this + " cannot inherit from it"); + } + } + superType = type; + } + + public Collection<StructDataType> getInheritedTypes() { + if (superType == null) { + return ImmutableList.of(); + } + return ImmutableList.of(superType); + } + + public boolean inherits(StructDataType type) { + if (equals(type)) return true; + if (superType != null && superType.inherits(type)) return true; + return false; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof StructDataType)) return false; + if (!super.equals(o)) return false; + + StructDataType that = (StructDataType) o; + if (superType != null ? !superType.equals(that.superType) : that.superType != null) return false; + return true; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (superType != null ? superType.hashCode() : 0); + return result; + } +} diff --git a/document/src/main/java/com/yahoo/document/StructuredDataType.java b/document/src/main/java/com/yahoo/document/StructuredDataType.java new file mode 100644 index 00000000000..18318913742 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/StructuredDataType.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.vespa.objects.Ids; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * TODO: What is this and why + * + * @author <a href="mailto:humbe@yahoo-inc.com">HÃ¥kon Humberset</a> + */ +public abstract class StructuredDataType extends DataType { + + public static final int classId = registerClass(Ids.document + 56, StructuredDataType.class); + + protected static int createId(String name) { + if (name.equals("document")) return 8; + + // This is broken really because we now depend on String.hashCode staying the same in Java vm's + // which is likely for pragmatic reasons but not by contract + return (name+".0").hashCode(); // the ".0" must be preserved to keep hashCodes the same after we removed version + } + + public StructuredDataType(String name) { + super(name, createId(name)); + } + + public StructuredDataType(int id, String name) { + super(name, id); + } + + @Override + public abstract StructuredFieldValue createFieldValue(); + + @Override + protected FieldValue createByReflection(Object arg) { return null; } + + /** + * Returns the name of this as a DataTypeName + * + * @return Return the Documentname of this doumenttype. + */ + public DataTypeName getDataTypeName() { + return new DataTypeName(getName()); + } + + /** + * Gets the field matching a given name. + * + * @param name The name of a field. + * @return Returns the matching field, or null if not found. + */ + public abstract Field getField(String name); + + /** + * Gets the field with the specified id. + * + * @param id the id of the field to return. + * @return the matching field, or null if not found. + */ + public abstract Field getField(int id); + + public abstract Collection<Field> getFields(); + + @Override + public boolean equals(Object o) { + return ((o instanceof StructuredDataType) && super.equals(o)); + } + + @Override + public int hashCode() { + return super.hashCode(); + } + + @Override + protected void register(DocumentTypeManager manager, List<DataType> seenTypes) { + seenTypes.add(this); + for (Field field : getFields()) { + if (!seenTypes.contains(field.getDataType())) { + //we haven't seen this one before, register it: + field.getDataType().register(manager, seenTypes); + } + } + super.register(manager, seenTypes); + } + + @Override + public FieldPath buildFieldPath(String remainFieldName) { + if (remainFieldName.length() == 0) { + return new FieldPath(); + } + + String currFieldName = remainFieldName; + String subFieldName = ""; + + for (int i = 0; i < remainFieldName.length(); i++) { + if (remainFieldName.charAt(i) == '.') { + currFieldName = remainFieldName.substring(0, i); + subFieldName = remainFieldName.substring(i + 1); + break; + } else if (remainFieldName.charAt(i) == '{' || remainFieldName.charAt(i) == '[') { + currFieldName = remainFieldName.substring(0, i); + subFieldName = remainFieldName.substring(i); + break; + } + } + + Field f = getField(currFieldName); + if (f != null) { + FieldPath fieldPath = f.getDataType().buildFieldPath(subFieldName); + List<FieldPathEntry> tmpPath = new ArrayList<FieldPathEntry>(fieldPath.getList()); + tmpPath.add(0, FieldPathEntry.newStructFieldEntry(f)); + return new FieldPath(tmpPath); + } else { + throw new IllegalArgumentException("Field '" + currFieldName + "' not found in type " + this); + } + } + +} diff --git a/document/src/main/java/com/yahoo/document/TemporaryDataType.java b/document/src/main/java/com/yahoo/document/TemporaryDataType.java new file mode 100644 index 00000000000..da65dde72da --- /dev/null +++ b/document/src/main/java/com/yahoo/document/TemporaryDataType.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.FieldValue; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +class TemporaryDataType extends DataType { + TemporaryDataType(int dataTypeId) { + super("temporary_" + dataTypeId, dataTypeId); + } + + @Override + public FieldValue createFieldValue() { + return null; + } + + @Override + public Class getValueClass() { + return null; + } + + @Override + public boolean isValueCompatible(FieldValue value) { + return false; + } +} diff --git a/document/src/main/java/com/yahoo/document/TemporaryStructuredDataType.java b/document/src/main/java/com/yahoo/document/TemporaryStructuredDataType.java new file mode 100644 index 00000000000..51cc6f94f2e --- /dev/null +++ b/document/src/main/java/com/yahoo/document/TemporaryStructuredDataType.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +/** + * Internal class, DO NOT USE!! Only public because it must be used from com.yahoo.searchdefinition.parser. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class TemporaryStructuredDataType extends StructDataType { + TemporaryStructuredDataType(String name) { + super(name); + } + + public static TemporaryStructuredDataType create(String name) { + return new TemporaryStructuredDataType(name); + } + + @Override + protected void setName(String name) { + super.setName(name); + setId(createId(getName())); + } +} diff --git a/document/src/main/java/com/yahoo/document/TestAndSetCondition.java b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java new file mode 100644 index 00000000000..5c0d83678c8 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.google.common.annotations.Beta; + +import java.util.Optional; + +/** + * The TestAndSetCondition class represents a test and set condition. + * A test and set condition is an (optional) string representing a + * document selection (cf. document selection language), which is used + * to match a document for test and set. If #isPresent evaluates to false, + * the condition is not present and matches any document. + * + * @author Vegard Sjonfjell + */ +@Beta +public class TestAndSetCondition { + public static final TestAndSetCondition NOT_PRESENT_CONDITION = new TestAndSetCondition(); + + private final String conditionStr; + + public TestAndSetCondition() { + this(""); + } + + public TestAndSetCondition(String conditionStr) { + this.conditionStr = conditionStr; + } + + public String getSelection() { return conditionStr; } + + public boolean isPresent() { return !conditionStr.isEmpty(); } + + /** + * Maps and optional test and set conditiong string to a TestAndSetCondition. + * If the condition string is not present, a "not present" condition is returned + * @param conditionString test and set conditiong string (document selection) + * @return a TestAndSetCondition representing the condition string or a "not present" condition + */ + public static TestAndSetCondition fromConditionString(Optional<String> conditionString) { + return conditionString + .map(TestAndSetCondition::new) + .orElse(TestAndSetCondition.NOT_PRESENT_CONDITION); + } +} diff --git a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java new file mode 100644 index 00000000000..9674d39fea8 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document; + +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.vespa.objects.Ids; +import com.yahoo.vespa.objects.ObjectVisitor; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class WeightedSetDataType extends CollectionDataType { + // The global class identifier shared with C++. + public static int classId = registerClass(Ids.document + 55, WeightedSetDataType.class); + + /** Should an arith operation to a non-existant member of a weightedset cause the member to be created */ + private boolean createIfNonExistent = false; + + /** Should a member of a weightedset with weight 0 be removed */ + private boolean removeIfZero = false; + + /** The tag type is ambiguous, this flag is true if the user explicitly set a field to tag */ + private boolean tag = false; + + public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) { + this(nestedType, createIfNonExistent, removeIfZero, 0); + if ((nestedType == STRING) && createIfNonExistent && removeIfZero) { + setId(18); + } else { + setId(getName().toLowerCase().hashCode()); + } + } + + public WeightedSetDataType(DataType nestedType, boolean createIfNonExistent, boolean removeIfZero, int id) { + super(createName(nestedType, createIfNonExistent, removeIfZero), id, nestedType); + this.createIfNonExistent = createIfNonExistent; + this.removeIfZero = removeIfZero; + } + + public WeightedSetDataType(String typeName, int code, DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) { + super(typeName != null ? createName(nestedType, createIfNonExistent, removeIfZero) : null, code, nestedType); + if ((code >= 0) && (code <= DataType.lastPredefinedDataTypeId()) && (code != 18)) // 18 == DataType.TAG.getId() is not yet initialized + throw new IllegalArgumentException("Cannot create a weighted set datatype with code " + code); + this.createIfNonExistent = createIfNonExistent; + this.removeIfZero = removeIfZero; + } + + @Override + public WeightedSetDataType clone() { + return (WeightedSetDataType) super.clone(); + } + + /** + * Called by SD parser if a data type is explicitly tag. + * @param tag True if this is a tag set. + */ + public void setTag(boolean tag) { + this.tag = tag; + } + + /** + * Returns whether or not this is a <em>tag</em> type weighted set. + * @return True if this is a tag set. + */ + public boolean isTag() { + return tag; + } + + static private String createName(DataType nested, boolean createIfNonExistant, boolean removeIfZero) { + if (nested == DataType.STRING && createIfNonExistant && removeIfZero) { + return "tag"; + } else { + String name = "WeightedSet<" + nested.getName() + ">"; + if (createIfNonExistant) name += ";Add"; + if (removeIfZero) name += ";Remove"; + return name; + } + } + + @Override + public WeightedSet createFieldValue() { + return new WeightedSet(this); + } + + @Override + public Class getValueClass() { + return WeightedSet.class; + } + + /** + * Returns true if this has the property createIfNonExistent (only relevant for weighted sets) + * + * @return createIfNonExistent property + */ + public boolean createIfNonExistent() { + return createIfNonExistent; + } + + /** + * Returns true if this has the property removeIfZero (only relevant for weighted sets) + * + * @return removeIfZero property + */ + public boolean removeIfZero() { + return removeIfZero; + } + @Override + public void visitMembers(ObjectVisitor visitor) { + super.visitMembers(visitor); + visitor.visit("removeIfZero", removeIfZero); + visitor.visit("createIfNonExistent", createIfNonExistent); + } + + @Override + public FieldPath buildFieldPath(String remainFieldName) + { + return MapDataType.buildFieldPath(remainFieldName, getNestedType(), DataType.INT); + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/AlternateSpanList.java b/document/src/main/java/com/yahoo/document/annotation/AlternateSpanList.java new file mode 100644 index 00000000000..9f2f8a9b160 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/AlternateSpanList.java @@ -0,0 +1,634 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.yahoo.document.serialization.SpanNodeReader; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; + +/** + * A node in a {@link SpanNode} tree that can have a <strong>multiple</strong> trees of child nodes, each with its own probability. + * This class has quite a few convenience methods for accessing the <strong>first</strong> subtree. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @see com.yahoo.document.annotation.SpanList + */ +public class AlternateSpanList extends SpanList { + public static final byte ID = 4; + private final List<Children> childTrees = new LinkedList<Children>(); + private static final Comparator<Children> childComparator = new ProbabilityComparator(); + + /** Create a new AlternateSpanList instance, having a single subtree with probability 1.0. */ + public AlternateSpanList() { + super((List<SpanNode>) null); + ensureAtLeastOneSubTree(); + } + + /* + * Deep-copies another AlternateSpanList. + * + * @param otherSpanList the instance to deep-copy. + */ + public AlternateSpanList(AlternateSpanList otherSpanList) { + super((List<SpanNode>) null); + for (Children otherSubtree : otherSpanList.childTrees) { + //create our own subtree: + Children children = new Children(this); + //copy nodes: + for (SpanNode otherNode : otherSubtree.children()) { + if (otherNode instanceof Span) { + children.add(new Span((Span) otherNode)); + } else if (otherNode instanceof AlternateSpanList) { + children.add(new AlternateSpanList((AlternateSpanList) otherNode)); + } else if (otherNode instanceof SpanList) { + children.add(new SpanList((SpanList) otherNode)); + } else if (otherNode instanceof DummySpanNode) { + children.add(otherNode); //shouldn't really happen + } else { + throw new IllegalStateException("Cannot create copy of " + otherNode + " with class " + + ((otherNode == null) ? "null" : otherNode.getClass())); + } + } + //add this subtree to our subtrees: + childTrees.add(children); + } + } + + public AlternateSpanList(SpanNodeReader reader) { + this(); + reader.read(this); + } + + private void ensureAtLeastOneSubTree() { + if (childTrees.isEmpty()) { + childTrees.add(new Children(getParent())); + } + } + + /** + * Adds a child node to the <strong>first</strong> subtree of this AlternateSpanList. Note + * that it might be a good idea to call {@link #sortSubTreesByProbability()} first. + * + * @param node the node to add. + * @return this, for call chaining + */ + @Override + public AlternateSpanList add(SpanNode node) { + return add(0, node); + } + + /** + * Sorts the subtrees under this AlternateSpanList by descending probability, such that the most probable + * subtree becomes the first subtree, and so on. + */ + public void sortSubTreesByProbability() { + resetCachedFromAndTo(); + Collections.sort(childTrees, childComparator); + } + + /** + * Returns a modifiable {@link List} of child nodes of <strong>first</strong> subtree. + * + * @return a modifiable {@link List} of child nodes of <strong>first</strong> subtree + */ + @Override + protected List<SpanNode> children() { + return children(0); + } + + /** + * Returns the number of subtrees under this node. + * + * @return the number of subtrees under this node. + */ + public int getNumSubTrees() { + return childTrees.size(); + } + + /** Clears all subtrees (the subtrees themselves are kept, but their contents are cleared and become invalidated). */ + @Override + public void clearChildren() { + for (Children c : childTrees) { + c.clearChildren(); + } + } + + /** + * Clears a given subtree (the subtree itself is kept, but its contents are cleared and become invalidated). + * + * @param i the index of the subtree to clear + */ + public void clearChildren(int i) { + Children c = childTrees.get(i); + if (c != null) { + c.clearChildren(); + } + } + + /** + * Sorts children in <strong>all</strong> subtrees by occurrence in the text covered. + * + * @see SpanNode#compareTo(SpanNode) + */ + @Override + public void sortChildren() { + for (Children children : childTrees) { + Collections.sort(children.children()); + } + } + + /** + * Sorts children in subtree i by occurrence in the text covered. + * + * @param i the index of the subtree to sort + * @see SpanNode#compareTo(SpanNode) + */ + public void sortChildren(int i) { + Children children = childTrees.get(i); + Collections.sort(children.children()); + } + + /** + * Recursively sorts all children in <strong>all</strong> subtrees by occurrence in the text covered. + */ + public void sortChildrenRecursive() { + for (Children children : childTrees) { + for (SpanNode node : children.children()) { + if (node instanceof SpanList) { + ((SpanList) node).sortChildrenRecursive(); + } + } + Collections.sort(children.children()); + } + } + + /** + * Recursively sorts all children in subtree i by occurrence in the text covered. + * + * @param i the index of the subtree to sort recursively + */ + public void sortChildrenRecursive(int i) { + Children children = childTrees.get(i); + for (SpanNode node : children.children()) { + if (node instanceof SpanList) { + ((SpanList) node).sortChildrenRecursive(); + } + } + Collections.sort(children.children()); + } + + + /** + * Moves a child of this SpanList to another SpanList. + * + * @param i the index of the subtree to remove the node from + * @param node the node to move + * @param target the SpanList to add the node to + * @throws IllegalArgumentException if the given node is not a child of this SpanList + */ + public void move(int i, SpanNode node, SpanList target) { + boolean removed = children(i).remove(node); + if (removed) { + //we found the node + node.setParent(null); + resetCachedFromAndTo(); + target.add(node); + } else { + throw new IllegalArgumentException("Node " + node + " is not a child of this SpanList, cannot move."); + } + } + + /** + * Moves a child of this SpanList to another SpanList. + * + * @param i the index of the subtree to remove the node from + * @param nodeNum the index of the node to move + * @param target the SpanList to add the node to + * @throws IndexOutOfBoundsException if the given index is out of range + */ + public void move(int i, int nodeNum, SpanList target) { + SpanNode node = children(i).remove(nodeNum); + if (node != null) { + //we found the node + node.setParent(null); + resetCachedFromAndTo(); + target.add(node); + } + } + + /** + * Moves a child of this SpanList to another SpanList. + * + * @param i the index of the subtree to remove the node from + * @param node the node to move + * @param target the SpanList to add the node to + * @param targetSubTree the index of the subtree of the given AlternateSpanList to add the node to + * @throws IllegalArgumentException if the given node is not a child of this SpanList + * @throws IndexOutOfBoundsException if the given index is out of range, or if the target subtree index is out of range + */ + public void move(int i, SpanNode node, AlternateSpanList target, int targetSubTree) { + if (targetSubTree < 0 || targetSubTree >= target.getNumSubTrees()) { + throw new IndexOutOfBoundsException(target + " has no subtree at index " + targetSubTree); + } + boolean removed = children(i).remove(node); + if (removed) { + //we found the node + node.setParent(null); + resetCachedFromAndTo(); + target.add(targetSubTree, node); + } else { + throw new IllegalArgumentException("Node " + node + " is not a child of this SpanList, cannot move."); + } + } + + /** + * Moves a child of this SpanList to another SpanList. + * + * @param i the index of the subtree to remove the node from + * @param nodeNum the index of the node to move + * @param target the SpanList to add the node to + * @param targetSubTree the index of the subtree of the given AlternateSpanList to add the node to + * @throws IndexOutOfBoundsException if any of the given indeces are out of range, or the target subtree index is out of range + */ + public void move(int i, int nodeNum, AlternateSpanList target, int targetSubTree) { + if (targetSubTree < 0 || targetSubTree >= target.getNumSubTrees()) { + throw new IndexOutOfBoundsException(target + " has no subtree at index " + targetSubTree); + } + SpanNode node = children(i).remove(nodeNum); + if (node != null) { + //we found the node + node.setParent(null); + resetCachedFromAndTo(); + target.add(targetSubTree, node); + } + } + + + /** + * Traverses all immediate children of all subtrees of this AlternateSpanList. + * The ListIterator only supports iteration forwards, and the optional operations that are implemented are + * remove() and set(). add() is not supported. + * + * @return a ListIterator which traverses all immediate children of this SpanNode + * @see java.util.ListIterator + */ + @Override + public ListIterator<SpanNode> childIterator() { + List<ListIterator<SpanNode>> childIterators = new ArrayList<ListIterator<SpanNode>>(); + for (Children ch : childTrees) { + childIterators.add(ch.childIterator()); + } + return new SerialIterator(childIterators); + } + + /** + * Recursively traverses all children (not only leaf nodes) of all subtrees of this AlternateSpanList, in a + * depth-first fashion. + * The ListIterator only supports iteration forwards, and the optional operations that are implemented are + * remove() and set(). add() is not supported. + * + * @return a ListIterator which recursively traverses all children and their children etc. of all subtrees of this AlternateSpanList + * @see java.util.ListIterator + */ + @Override + public ListIterator<SpanNode> childIteratorRecursive() { + List<ListIterator<SpanNode>> childIterators = new ArrayList<ListIterator<SpanNode>>(); + for (Children ch : childTrees) { + childIterators.add(ch.childIteratorRecursive()); + } + return new SerialIterator(childIterators); + } + + /** + * Traverses all immediate children of the given subtree of this AlternateSpanList. + * The ListIterator returned supports all optional operations + * specified in the ListIterator interface. + * + * @param i the index of the subtree to iterate over + * @return a ListIterator which traverses all immediate children of this SpanNode + * @see java.util.ListIterator + */ + public ListIterator<SpanNode> childIterator(int i) { + return childTrees.get(i).childIterator(); + } + + /** + * Recursively traverses all children (not only leaf nodes) of the given subtree of this AlternateSpanList, in a + * depth-first fashion. + * The ListIterator only supports iteration forwards, and the optional operations that are implemented are + * remove() and set(). add() is not supported. + * + * @param i the index of the subtree to iterate over + * @return a ListIterator which recursively traverses all children and their children etc. of the given subtree of this AlternateSpanList. + * @see java.util.ListIterator + */ + public ListIterator<SpanNode> childIteratorRecursive(int i) { + return childTrees.get(i).childIteratorRecursive(); + } + + public int numChildren(int i) { + return children(i).size(); + } + + + /** + * Returns a modifiable {@link List} of child nodes of the specified subtree. + * + * @param i the index of the subtree to search + * @return a modifiable {@link List} of child nodes of the specified subtree + */ + protected List<SpanNode> children(int i) { + return childTrees.get(i).children(); + } + + + @Override + void setParent(SpanNodeParent parent) { + super.setParent(parent); + for (Children ch : childTrees) { + ch.setParent(parent); + } + } + + /** + * Adds a possible subtree of this AlternateSpanList, with the given probability. Note that the first subtree is + * always available through the use of children(), so this method is only used for adding the second or higher + * subtree. + * + * @param subtree the subtree to add + * @param probability the probability of this subtree + * @return true if successful + * @see #children() + */ + public boolean addChildren(List<SpanNode> subtree, double probability) { + Children childTree = new Children(getParent(), subtree, probability); + resetCachedFromAndTo(); + return childTrees.add(childTree); + + } + + /** + * Adds a possible subtree of this AlternateSpanList, with the given probability, at index i. Note that the first subtree is + * always available through the use of children(), so this method is only used for adding the second or higher + * subtree. + * + * @param i the index of where to insert the subtree + * @param subtree the subtree to add + * @param probability the probability of this subtree + * @see #children() + */ + public void addChildren(int i, List<SpanNode> subtree, double probability) { + Children childTree = new Children(getParent(), subtree, probability); + resetCachedFromAndTo(); + childTrees.add(i, childTree); + } + + /** + * Removes the subtree at index i (both the subtree itself and its contents, which become invalidated). + * Note that if this AlternateSpanList has only one subtree and index 0 is given, + * a new empty subtree is automatically added, since an AlternateSpanList always has at least one subtree. + * + * @param i the index of the subtree to remove + * @return the subtree removed, if any (note: invalidated) + */ + public List<SpanNode> removeChildren(int i) { + Children retval = childTrees.remove(i); + ensureAtLeastOneSubTree(); + resetCachedFromAndTo(); + if (retval != null) { + retval.setInvalid(); + retval.setParent(null); + for (SpanNode node : retval.children()) { + node.setParent(null); + } + return retval.children(); + } + return null; + } + + /** + * Removes all subtrees (both the subtrees themselves and their contents, which become invalidated). + * Note that a new empty subtree is automatically added at index 0, since an AlternateSpanList always has at + * least one subtree. + */ + public void removeChildren() { + for (Children ch : childTrees) { + ch.setInvalid(); + ch.setParent(null); + ch.clearChildren(); + } + childTrees.clear(); + resetCachedFromAndTo(); + ensureAtLeastOneSubTree(); + } + + @Override + void setInvalid() { + //invalidate ourselves: + super.setInvalid(); + //invalidate all child trees + for (Children ch : childTrees) { + ch.setInvalid(); + } + } + + + /** + * Sets the subtree at index i. + * + * @param i the index of where to set the subtree + * @param subtree the subtree to set + * @param probability the probability to set + * @return the overwritten subtree, if any + */ + public List<SpanNode> setChildren(int i, List<SpanNode> subtree, double probability) { + resetCachedFromAndTo(); + if (childTrees.size() == 1 && i == 0) { + //replace the first subtree + Children sub = new Children(getParent(), subtree, probability); + Children retval = childTrees.set(i, sub); + if (retval == null) { + return null; + } else { + retval.setParent(null); + for (SpanNode node : retval.children()) { + node.setParent(null); + } + return retval.children(); + } + } + List<SpanNode> retval = removeChildren(i); + addChildren(i, subtree, probability); + return retval; + } + + /** + * Returns the character index where this {@link SpanNode} starts (inclusive), i.e. the smallest {@link com.yahoo.document.annotation.SpanNode#getFrom()} of all children in subtree i. + * + * @param i the index of the subtree to use + * @return the lowest getFrom() of all children in subtree i, or -1 if this SpanList has no children in subtree i. + * @throws IndexOutOfBoundsException if this AlternateSpanList has no subtree i + */ + public int getFrom(int i) { + return childTrees.get(i).getFrom(); + } + + + /** + * Returns the character index where this {@link SpanNode} ends (exclusive), i.e. the greatest {@link com.yahoo.document.annotation.SpanNode#getTo()} of all children in subtree i. + * + * @param i the index of the subtree to use + * @return the greatest getTo() of all children, or -1 if this SpanList has no children in subtree i. + * @throws IndexOutOfBoundsException if this AlternateSpanList has no subtree i + */ + public int getTo(int i) { + return childTrees.get(i).getTo(); + } + + /** + * Returns the length of this span according to subtree i, i.e. getFrom(i) - getTo(i). + * + * @param i the index of the subtree to use + * @return the length of this span according to subtree i + */ + public int getLength(int i) { + return getTo(i) - getFrom(i); + } + + /** + * Returns the text covered by this span as given by subtree i, or null if subtree i is empty. + * + * @param i the index of the subtree to use + * @param text the text to get a substring from + * @return the text covered by this span as given by subtree i, or null if subtree i is empty + */ + public CharSequence getText(int i, CharSequence text) { + if (children(i).isEmpty()) { + return null; + } + StringBuilder str = new StringBuilder(); + List<SpanNode> ch = children(i); + for (SpanNode node : ch) { + CharSequence childText = node.getText(text); + if (childText != null) { + str.append(node.getText(text)); + } + } + return str; + } + + /** + * Returns the probability of the given subtree. + * + * @param i the subtree to return the probability of + * @return the probability of the given subtree + */ + public double getProbability(int i) { + return childTrees.get(i).getProbability(); + } + + /** + * Sets the probability of the given subtree. + * + * @param i the subtree to set the probability of + * @param probability the probability to set + */ + public void setProbability(int i, double probability) { + childTrees.get(i).setProbability(probability); + } + + /** Normalizes all probabilities between 0.0 (inclusive) and 1.0 (exclusive). */ + public void normalizeProbabilities() { + double sum = 0.0; + for (Children c : childTrees) { + sum += c.getProbability(); + } + double coeff = 1.0 / sum; + + for (Children childTree : childTrees) { + double newProb = childTree.getProbability() * coeff; + childTree.setProbability(newProb); + } + } + + + /** + * Convenience method to add a span node to the child tree at index i. This is equivalent to calling + * <code> + * AlternateSpanList.children(i).add(node); + * </code> + * + * @param i index + * @param node span node + */ + public AlternateSpanList add(int i, SpanNode node) { + checkValidity(node, children(i)); + node.setParent(this); + children(i).add(node); + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof AlternateSpanList)) return false; + if (!super.equals(o)) return false; + + AlternateSpanList that = (AlternateSpanList) o; + + if (!childTrees.equals(that.childTrees)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + childTrees.hashCode(); + return result; + } + + @Override + public String toString() { + return "AlternateSpanList, num subtrees=" + getNumSubTrees(); + } + + + private static class ProbabilityComparator implements Comparator<Children> { + @Override + public int compare(Children o1, Children o2) { + return Double.compare(o2.probability, o1.probability); //note: opposite of natural ordering! + } + } + + private class Children extends SpanList { + private double probability = 1.0; + + private Children(SpanNodeParent parent) { + setParent(parent); + } + + private Children(SpanNodeParent parent, List<SpanNode> children, double probability) { + super(children); + setParent(parent); + if (children != null) { + for (SpanNode node : children) { + node.setParent(AlternateSpanList.this); + } + } + this.probability = probability; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double probability) { + this.probability = probability; + } + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/Annotation.java b/document/src/main/java/com/yahoo/document/annotation/Annotation.java new file mode 100644 index 00000000000..1823584eb6d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/Annotation.java @@ -0,0 +1,260 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; + +/** + * An Annotation describes some kind of information associated with + * a {@link SpanNode}. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @see com.yahoo.document.annotation.SpanNode + * @see com.yahoo.document.annotation.AnnotationType + */ +public class Annotation implements Comparable<Annotation> { + private AnnotationType type; + private SpanNode spanNode = null; + private FieldValue value = null; + /** + * This scratch id is used to avoid using IdentityHashMaps as they are very costly. + */ + private int scratchId = -1; + public void setScratchId(int id) { + scratchId = id; + } + + public int getScratchId() { + return scratchId; + } + + + /** + * Constructs an Annotation without a type. BEWARE! Should only be used during deserialization. + */ + public Annotation() { + } + + /** + * Constructs a new annotation of the specified type. + * + * @param type the type of the new annotation + */ + public Annotation(AnnotationType type) { + this.type = type; + } + + /** + * Constructs a copy of the input annotation. + * + * @param other annotation + */ + public Annotation(Annotation other) { + this.type = other.type; + this.value = ((other.value == null) ? null : other.value.clone()); + //do not copy spanNode now + } + + /** + * Constructs a new annotation of the specified type, and having the specified value. + * + * @param type the type of the new annotation + * @param value the value of the new annotation + * @throws UnsupportedOperationException if the annotation type does not allow this annotation to have values. + */ + public Annotation(AnnotationType type, FieldValue value) { + this(type); + setFieldValue(value); + } + + /** + * Returns the type of this annotation. + * + * @return the type of this annotation + */ + public AnnotationType getType() { + return type; + } + + /** + * Sets the type of this annotation. BEWARE! Should only be used during deserialization. + * + * @param type the type of this annotation + */ + public void setType(AnnotationType type) { + this.type = type; + } + + /** + * Returns true if this Annotation is associated with a SpanNode (irrespective of the SpanNode being valid or not). + * + * @return true if this Annotation is associated with a SpanNode, false otherwise. + * @see com.yahoo.document.annotation.SpanNode#isValid() + */ + public boolean hasSpanNode() { + return spanNode != null; + } + + /** + * Returns true iff. this Annotation is associated with a SpanNode and the SpanNode is valid. + * + * @return true iff. this Annotation is associated with a SpanNode and the SpanNode is valid. + * @see com.yahoo.document.annotation.SpanNode#isValid() + */ + public boolean isSpanNodeValid() { + return spanNode != null && spanNode.isValid(); + } + + + /** + * Returns the {@link SpanNode} this Annotation is annotating, if any. + * + * @return the {@link SpanNode} this Annotation is annotating, or null + * @throws IllegalStateException if this Annotation is associated with a SpanNode and the SpanNode is invalid. + */ + public SpanNode getSpanNode() { + if (spanNode != null && !spanNode.isValid()) { + throw new IllegalStateException("Span node is invalid: " + spanNode); + } + return spanNode; + } + + /** + * Returns the {@link SpanNode} this Annotation is annotating, if any. + * + * @return the {@link SpanNode} this Annotation is annotating, or null + */ + public final SpanNode getSpanNodeFast() { + return spanNode; + } + + /** + * WARNING! Should only be used by deserializers! Sets the span node that this annotation points to. + * + * @param spanNode the span node that this annotation shall point to. + */ + public void setSpanNode(SpanNode spanNode) { + if (this.spanNode != null && spanNode != null) { + throw new IllegalStateException("WARNING! " + this + " is already attached to node " + this.spanNode + + ". Attempt to attach to node " + spanNode + + ". Annotation instances MUST NOT be shared among SpanNodes."); + } + if (spanNode != null && !spanNode.isValid()) { + throw new IllegalStateException("Span node is invalid: " + spanNode); + } + if (spanNode == DummySpanNode.INSTANCE) { + //internal safeguard + throw new IllegalStateException("BUG!! Annotations should never be attached to DummySpanNode."); + } + this.spanNode = spanNode; + } + + /** + * WARNING! Should only be used by deserializers! Sets the span node that this annotation points to. + * + * @param spanNode the span node that this annotation shall point to. + */ + public final void setSpanNodeFast(SpanNode spanNode) { + this.spanNode = spanNode; + } + + /** + * Returns the value of the annotation, if any. + * + * @return the value of the annotation, or null + */ + public FieldValue getFieldValue() { + return value; + } + + /** + * Sets the value of the annotation. + * + * @param fieldValue the value to set + * @throws UnsupportedOperationException if the annotation type does not allow this annotation to have values. + */ + public void setFieldValue(FieldValue fieldValue) { + if (fieldValue == null) { + value = null; + return; + } + + DataType type = getType().getDataType(); + if (type != null && type.isValueCompatible(fieldValue)) { + this.value = fieldValue; + } else { + String typeName = (type == null) ? "null" : type.getValueClass().getName(); + throw new IllegalArgumentException("Argument is of wrong type, must be of type " + typeName + + ", was " + fieldValue.getClass().getName()); + } + } + + /** + * Returns true if this Annotation has a value. + * + * @return true if this Annotation has a value, false otherwise. + */ + public boolean hasFieldValue() { + return value != null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Annotation)) return false; + + Annotation that = (Annotation) o; + if (!type.equals(that.type)) return false; + if (spanNode != null ? !spanNode.equals(that.spanNode) : that.spanNode != null) return false; + if (value != null ? !value.equals(that.value) : that.value != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = type.hashCode(); + result = 31 * result + (spanNode != null ? spanNode.hashCode() : 0); + result = 31 * result + (value != null ? value.toString().hashCode() : 0); + return result; + } + + @Override + public String toString() { + String retval = "annotation of type " + type; + retval += ((value == null) ? " (no value)" : " (with value)"); + return retval; + } + + + @Override + public int compareTo(Annotation annotation) { + int comp; + + if (spanNode == null) { + comp = (annotation.spanNode == null) ? 0 : -1; + } else { + comp = (annotation.spanNode == null) ? 1 : spanNode.compareTo(annotation.spanNode); + } + + if (comp != 0) { + return comp; + } + + comp = type.compareTo(annotation.type); + + if (comp != 0) { + return comp; + } + + //types are equal, too, compare values + if (value == null) { + comp = (annotation.value == null) ? 0 : -1; + } else { + comp = (annotation.value == null) ? 1 : value.compareTo(annotation.value); + } + + return comp; + } +} + diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java new file mode 100644 index 00000000000..3a44d2a78f6 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import java.util.Collection; +import java.util.Iterator; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +abstract class AnnotationContainer { + + /** + * Adds all annotations of the given collection to this container. + * + * @param annotations the annotations to add. + */ + abstract void annotateAll(Collection<Annotation> annotations); + + /** + * Adds an annotation to this container. + * + * @param annotation the annotation to add. + */ + abstract void annotate(Annotation annotation); + + /** + * Returns a mutable collection of annotations. + * + * @return a mutable collection of annotations. + */ + abstract Collection<Annotation> annotations(); + + /** + * Returns an Iterator over all annotations that annotate the given node. + * + * @param node the node to return annotations for. + * @return an Iterator over all annotations that annotate the given node. + */ + abstract Iterator<Annotation> iterator(SpanNode node); + + /** + * Returns a recursive Iterator over all annotations that annotate the given node and its subnodes. + * + * @param node the node to recursively return annotations for. + * @return a recursive Iterator over all annotations that annotate the given node and its subnodes. + */ + abstract Iterator<Annotation> iteratorRecursive(SpanNode node); + + + //TODO: remember equals and hashcode in subclasses! +} diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationReference.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationReference.java new file mode 100644 index 00000000000..aa6ea1b0040 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationReference.java @@ -0,0 +1,185 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +/** + * A FieldValue which holds a reference to an annotation of a specified type. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @see Annotation#setFieldValue(com.yahoo.document.datatypes.FieldValue) + */ +public class AnnotationReference extends FieldValue { + + public static int classId = registerClass(Ids.annotation + 2, AnnotationReference.class); + private Annotation reference; + private AnnotationReferenceDataType dataType; + + /** + * Constructs a new AnnotationReference, with a reference to the given {@link Annotation}. + * + * @param type the data type of this AnnotationReference + * @param reference the reference to set + * @throws IllegalArgumentException if the given annotation has a type that is not compatible with this reference + */ + public AnnotationReference(AnnotationReferenceDataType type, Annotation reference) { + this.dataType = type; + setReference(reference); + } + + /** + * Constructs a new AnnotationReference. + * + * @param type the data type of this AnnotationReference + */ + public AnnotationReference(AnnotationReferenceDataType type) { + this(type, null); + } + + /** + * Clones this AnnotationReference. Note: No deep-copying, so the AnnotationReference returned + * refers to the same Annotation as this AnnotationReference. + * + * @return a copy of this object, referring to the same Annotation instance. + */ + @Override + public AnnotationReference clone() { + return (AnnotationReference) super.clone(); + //do not clone annotation that we're referring to. See wizardry in SpanTree for that. + } + + /** + * Returns the Annotation that this AnnotationReference refers to. + * + * @return the Annotation that this AnnotationReference refers to. + */ + public Annotation getReference() { + return reference; + } + + @Override + public void assign(Object o) { + if (o != null && (!(o instanceof Annotation))) { + throw new IllegalArgumentException("Cannot assign object of type " + o.getClass().getName() + " to an AnnotationReference, must be of type " + Annotation.class.getName()); + } + setReference((Annotation) o); + } + + /** + * Set an {@link Annotation} that this AnnotationReference shall refer to. + * + * @param reference an Annotation that this AnnotationReference shall refer to. + * @throws IllegalArgumentException if the given annotation has a type that is not compatible with this reference + */ + public void setReference(Annotation reference) { + if (reference == null) { + this.reference = null; + return; + } + AnnotationReferenceDataType type = getDataType(); + if (type.getAnnotationType().isValueCompatible(reference) + // The case if concrete annotation type + || reference.getType() instanceof AnnotationType) { + this.reference = reference; + } else { + throw new IllegalArgumentException("Cannot set reference, must be of type " + type + " (was of type " + reference.getType() + ")"); + } + } + + + /** + * WARNING! Only to be used by deserializers when reference is not fully deserialized yet! Sets + * an {@link Annotation} that this AnnotationReference shall refer to. + * + * @param reference an Annotation that this AnnotationReference shall refer to. + * @throws IllegalArgumentException if the given annotation has a type that is not compatible with this reference + */ + public void setReferenceNoCompatibilityCheck(Annotation reference) { + if (reference == null) { + this.reference = null; + return; + } + this.reference = reference; + } + + @Override + public AnnotationReferenceDataType getDataType() { + return dataType; + } + + public void setDataType(DataType dataType) { + if (dataType instanceof AnnotationReferenceDataType) { + this.dataType = (AnnotationReferenceDataType) dataType; + } else { + throw new IllegalArgumentException("Cannot set dataType to " + dataType + ", must be of type AnnotationReferenceDataType."); + } + } + + @Override + public void printXml(XmlStream xml) { + //TODO: Implement AnnotationReference.printXml() + } + + @Override + public void clear() { + this.reference = null; + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof AnnotationReference)) return false; + if (!super.equals(o)) return false; + + AnnotationReference that = (AnnotationReference) o; + + if (reference != null ? !reference.toString().equals(that.reference.toString()) : that.reference != null) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (reference != null ? reference.toString().hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "AnnotationReference " + getDataType() + " referring to " + reference; + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + if (comp == 0) { + //types are equal, this must be of this type + AnnotationReference value = (AnnotationReference) fieldValue; + if (reference == null) { + comp = (value.reference == null) ? 0 : -1; + } else { + comp = (value.reference == null) ? 1 : (reference.toString().compareTo(value.reference.toString())); + } + } + return comp; + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationReferenceDataType.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationReferenceDataType.java new file mode 100644 index 00000000000..e75e29f5e75 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationReferenceDataType.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; + +/** + * A data type describing a field value having a reference to an annotation of a given type. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class AnnotationReferenceDataType extends DataType { + private AnnotationType aType; + + /** + * Creates an AnnotationReferenceDataType with a generated id. + * + * @param aType the annotation type that AnnotationRefs shall refer to. + */ + public AnnotationReferenceDataType(AnnotationType aType) { + super("annotationreference<" + ((aType == null) ? "" : aType.getName()) + ">", 0); + setAnnotationType(aType); + } + + /** + * Creates an AnnotationReferenceDataType with a given id. + * + * @param aType the annotation type that AnnotationRefs shall refer to. + * @param id the id to use + */ + public AnnotationReferenceDataType(AnnotationType aType, int id) { + super("annotationreference<" + ((aType == null) ? "" : aType.getName()) + ">", id); + this.aType = aType; + } + + /** + * Creates an AnnotationReferenceDataType. WARNING! Do not use! + */ + protected AnnotationReferenceDataType() { + super("annotationreference<>", 0); + } + + private int createId() { + //TODO: This should be Java's hashCode(), since all other data types use it, and using something else here will probably lead to collisions + return getName().toLowerCase().hashCode(); + } + + @Override + public FieldValue createFieldValue() { + return new AnnotationReference(this); + } + + @Override + public Class getValueClass() { + return AnnotationReference.class; + } + + @Override + public boolean isValueCompatible(FieldValue value) { + if (!(value instanceof AnnotationReference)) { + return false; + } + AnnotationReference reference = (AnnotationReference) value; + if (equals(reference.getDataType())) { + return true; + } + return false; + } + + /** + * Returns the annotation type of this AnnotationReferenceDataType. + * + * @return the annotation type of this AnnotationReferenceDataType. + */ + public AnnotationType getAnnotationType() { + return aType; + } + + /** + * Sets the annotation type that this AnnotationReferenceDataType points to. WARNING! Do not use. + * @param type the annotation type of this AnnotationReferenceDataType. + */ + protected void setAnnotationType(AnnotationType type) { + this.aType = type; + setId(createId()); + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationType.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationType.java new file mode 100644 index 00000000000..f66080e3a5a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationType.java @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.google.common.collect.ImmutableList; +import com.yahoo.collections.MD5; +import com.yahoo.document.DataType; + +import java.util.Collection; +import java.util.Collections; + +/** + * An AnnotationType describes a certain type of annotations; they are + * generally distinguished by a name, an id, and an optional data type. + * <p> + * If an AnnotationType has a {@link com.yahoo.document.DataType}, this means that {@link Annotation}s of + * that type are allowed to have a {@link com.yahoo.document.datatypes.FieldValue} of the given + * {@link com.yahoo.document.DataType} as an optional payload. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class AnnotationType implements Comparable<AnnotationType> { + private final int id; + private final String name; + private DataType dataType; + private AnnotationType superType = null; + + /** + * Creates a new annotation type that cannot have values (hence no data type). + * + * @param name the name of the new annotation type + */ + public AnnotationType(String name) { + this(name, null); + } + + /** + * Creates a new annotation type that can have values of the specified type. + * + * @param name the name of the new annotation type + * @param dataType the data type of the annotation value + */ + public AnnotationType(String name, DataType dataType) { + this.name = name; + this.dataType = dataType; + //always keep this as last statement in here: + this.id = computeHash(); + } + + /** + * Creates a new annotation type that can have values of the specified type. + * + * @param name the name of the new annotation type + * @param dataType the data type of the annotation value + * @param id the ID of the new annotation type + */ + public AnnotationType(String name, DataType dataType, int id) { + this.name = name; + this.dataType = dataType; + this.id = id; + } + + /** + * Creates a new annotation type, with the specified ID. WARNING! Only to be used by configuration + * system, do not use!! + * + * @param name the name of the new annotation type + * @param id the ID of the new annotation type + */ + public AnnotationType(String name, int id) { + this.id = id; + this.name = name; + } + + /** + * Returns the name of this annotation. + * + * @return the name of this annotation. + */ + public String getName() { + return name; + } + + /** + * Returns the data type of this annotation, if any. + * + * @return the data type of this annotation, or null. + */ + public DataType getDataType() { + return dataType; + } + + /** + * Sets the data type of this annotation. WARNING! Only to be used by configuration + * system, do not use!! + * + * @param dataType the data type of the annotation value + */ + public void setDataType(DataType dataType) { + this.dataType = dataType; + } + + /** + * Returns the ID of this annotation. + * + * @return the ID of this annotation. + */ + public int getId() { + return id; + } + + private int computeHash() { + return new MD5().hash(name); + } + + public boolean isValueCompatible(Annotation structValue) { + if (structValue.getType().inherits(this)) { + //the value is of this type; or the supertype of the value is of this type, etc.... + return true; + } + return false; + } + + /** + * WARNING! Only to be used by the configuration system and in unit tests. Not to be used in production code. + * + * @param type the type to inherit from + */ + public void inherit(AnnotationType type) { + if (superType != null) { + throw new IllegalArgumentException("Already inherits type " + superType + ", multiple inheritance not currently supported."); + } + superType = type; + } + + public Collection<AnnotationType> getInheritedTypes() { + if (superType == null) { + return ImmutableList.of(); + } + return ImmutableList.of(superType); + } + + public boolean inherits(AnnotationType type) { + if (equals(type)) return true; + if (superType != null && superType.inherits(type)) return true; + return false; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof AnnotationType)) return false; + + AnnotationType that = (AnnotationType) o; + + return name.equals(that.name); + } + + @Override + public int hashCode() { + return id; + } + + @Override + public String toString() { + StringBuilder strb = new StringBuilder(); + strb.append(name).append(" (id ").append(id); + if (dataType != null) { + strb.append(", data type ").append(dataType); + } + strb.append(")"); + return strb.toString(); + } + + @Override + public int compareTo(AnnotationType annotationType) { + if (annotationType == null) { + return 1; + } + if (id < annotationType.id) { + return -1; + } else if (id > annotationType.id) { + return 1; + } + return 0; + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java new file mode 100644 index 00000000000..778e6f50a40 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import org.apache.commons.collections.map.MultiValueMap; + +import java.util.Collection; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +// TODO: Should this be removed? +public class AnnotationType2AnnotationContainer extends IteratingAnnotationContainer { + private final MultiValueMap annotationType2Annotation = MultiValueMap.decorate(new IdentityHashMap()); + + @Override + void annotateAll(Collection<Annotation> annotations) { + for (Annotation a : annotations) { + annotate(a); + } + } + + @Override + void annotate(Annotation annotation) { + annotationType2Annotation.put(annotation.getType(), annotation); + } + + @Override + @SuppressWarnings("unchecked") + Collection<Annotation> annotations() { + return annotationType2Annotation.values(); + } + + @Override + Iterator<Annotation> iterator(IdentityHashMap<SpanNode, SpanNode> nodes) { + return new NonRecursiveIterator(nodes); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof AnnotationType2AnnotationContainer)) return false; + AnnotationType2AnnotationContainer that = (AnnotationType2AnnotationContainer) o; + if (!annotationType2Annotation.equals(that.annotationType2Annotation)) return false; + return true; + } + + @Override + public int hashCode() { + return annotationType2Annotation.hashCode(); + } + + private class NonRecursiveIterator implements Iterator<Annotation> { + private final IdentityHashMap<SpanNode, SpanNode> nodes; + private final Iterator<Annotation> annotationIt; + private Annotation next = null; + private boolean nextCalled; + + @SuppressWarnings("unchecked") + public NonRecursiveIterator(IdentityHashMap<SpanNode, SpanNode> nodes) { + this.nodes = nodes; + this.annotationIt = annotationType2Annotation.values().iterator(); + } + + @Override + public boolean hasNext() { + if (next != null) { + return true; + } + while (annotationIt.hasNext()) { + Annotation tmp = annotationIt.next(); + if (nodes.containsKey(tmp.getSpanNodeFast())) { + next = tmp; + return true; + } + } + next = null; + return false; + } + + @Override + public Annotation next() { + if (hasNext()) { + Annotation tmp = next; + next = null; + nextCalled = true; + return tmp; + } + //there is no 'next' + throw new NoSuchElementException("No next element found."); + } + + @Override + public void remove() { + //only allowed to call remove immediately after next() + if (!nextCalled) { + //we have not next'ed the iterator, cannot do this: + throw new IllegalStateException("Cannot remove() before next()"); + } + annotationIt.remove(); + nextCalled = false; + } + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationTypeRegistry.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypeRegistry.java new file mode 100644 index 00000000000..21e0338ae0e --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypeRegistry.java @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * A registry of annotation types. This can be set up programmatically or from config. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class AnnotationTypeRegistry { + private Map<Integer, AnnotationType> idMap = new HashMap<Integer, AnnotationType>(); + private Map<String, AnnotationType> nameMap = new HashMap<String, AnnotationType>(); + + /** Creates a new empty registry. */ + public AnnotationTypeRegistry() { + } + + /** + * Register a new annotation type. WARNING! Only to be used by the configuration system and in unit tests. Not to be used in production code. + * + * @param type the type to register + * @throws IllegalArgumentException if a type is already registered with this name or this id, and it is non-equal to the argument. + */ + public void register(AnnotationType type) { + if (idMap.containsKey(type.getId()) || nameMap.containsKey(type.getName())) { + AnnotationType idType = idMap.get(type.getId()); + AnnotationType nameType = nameMap.get(type.getName()); + if (type.equals(idType) && type.equals(nameType)) { + //it's the same one being re-registered, we're OK! + return; + } + throw new IllegalArgumentException("A type is already registered with this name or this id."); + } + idMap.put(type.getId(), type); + nameMap.put(type.getName(), type); + } + + /** + * Unregisters the type given by the argument. WARNING! Only to be used by the configuration system and in unit tests. Not to be used in production code. + * + * @param name the name of the type to unregister. + * @return true if the type was successfully unregistered, false otherwise (it was not present) + */ + public boolean unregister(String name) { + AnnotationType oldType = nameMap.remove(name); + if (oldType != null) { + idMap.remove(oldType.getId()); + return true; + } + return false; + } + + /** + * Unregisters the type given by the argument. WARNING! Only to be used by the configuration system and in unit tests. Not to be used in production code. + * + * @param id the id of the type to unregister. + * @return true if the type was successfully unregistered, false otherwise (it was not present) + */ + public boolean unregister(int id) { + AnnotationType oldType = idMap.remove(id); + if (oldType != null) { + nameMap.remove(oldType.getName()); + return true; + } + return false; + } + + /** + * Unregisters the type given by the argument. WARNING! Only to be used by the configuration system and in unit tests. Not to be used in production code. + * + * @param type the AnnotationType to unregister. + * @return true if the type was successfully unregistered, false otherwise (it was not present) + * @throws IllegalArgumentException if the ID and name of this annotation type are present, but they do not belong together. + */ + public boolean unregister(AnnotationType type) { + if (idMap.containsKey(type.getId()) && nameMap.containsKey(type.getName())) { + AnnotationType idType = idMap.get(type.getId()); + AnnotationType nameType = nameMap.get(type.getName()); + + if (idType == nameType) { + //name and id belong together in our maps + idMap.remove(type.getId()); + nameMap.remove(type.getName()); + } else { + throw new IllegalArgumentException("The ID and name of this annotation type are present, but they do not belong together. Not removing type."); + } + return true; + } + //it's not there, but that's no problem + return false; + } + + /** + * Returns an annotation type with the given name. + * + * @param name the name of the annotation type to return + * @return an {@link AnnotationType} with the given name, or null if it is not registered + */ + public AnnotationType getType(String name) { + return nameMap.get(name); + } + + /** + * Returns an annotation type with the given id. + * + * @param id the id of the annotation type to return + * @return an {@link AnnotationType} with the given id, or null if it is not registered + */ + public AnnotationType getType(int id) { + return idMap.get(id); + } + + /** + * Returns an <strong>unmodifiable</strong> {@link Map} of all types registered. + * + * @return an unmodifiable {@link Map} of all types registered. + */ + public Map<String, AnnotationType> getTypes() { + return Collections.unmodifiableMap(nameMap); + } + + /** Clears all registered annotation types. */ + public void clear() { + idMap.clear(); + nameMap.clear(); + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java new file mode 100644 index 00000000000..6d868ed4079 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationTypes.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.yahoo.document.DataType; + +import java.util.Arrays; +import java.util.List; + +/** + * This is a container for all {@link Annotation}s constants used by built-in Vespa features. These must be in sync with + * the corresponding class in the C++ file 'document/datatype/annotationtype.h'. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@SuppressWarnings({ "UnusedDeclaration" }) +public final class AnnotationTypes { + + private AnnotationTypes() { + // unreachable + } + + public static final AnnotationType TERM = new AnnotationType("term", DataType.STRING, 1); + public static final AnnotationType TOKEN_TYPE = new AnnotationType("token_type", DataType.INT, 2); + public static final AnnotationType CANONICAL = new AnnotationType("canonical", DataType.STRING, 3); + public static final AnnotationType NORMALIZED = new AnnotationType("normalized", DataType.STRING, 4); + public static final AnnotationType READING = new AnnotationType("reading", DataType.STRING, 5); + public static final AnnotationType STEM = new AnnotationType("stem", DataType.STRING, 6); + public static final AnnotationType TRANSFORMED = new AnnotationType("transformed", DataType.STRING, 7); + public static final AnnotationType PROXIMITY_BREAK = new AnnotationType("proximity_break", DataType.DOUBLE, 8); + public static final AnnotationType SPECIAL_TOKEN = new AnnotationType("special_token", 9); + + public static final List<AnnotationType> ALL_TYPES = Arrays.asList(TERM, TOKEN_TYPE, CANONICAL, NORMALIZED, READING, + STEM, TRANSFORMED, PROXIMITY_BREAK, + SPECIAL_TOKEN); +} diff --git a/document/src/main/java/com/yahoo/document/annotation/DummySpanNode.java b/document/src/main/java/com/yahoo/document/annotation/DummySpanNode.java new file mode 100644 index 00000000000..c891e8c686b --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/DummySpanNode.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import java.util.Collections; +import java.util.ListIterator; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +class DummySpanNode extends SpanNode { + final static DummySpanNode INSTANCE = new DummySpanNode(); + + private DummySpanNode() { + } + + @Override + public boolean isLeafNode() { + return true; + } + + @Override + public ListIterator<SpanNode> childIterator() { + return Collections.<SpanNode>emptyList().listIterator(); + } + + @Override + public ListIterator<SpanNode> childIteratorRecursive() { + return Collections.<SpanNode>emptyList().listIterator(); + } + + @Override + public int getFrom() { + return 0; + } + + @Override + public int getTo() { + return 0; + } + + @Override + public int getLength() { + return 0; + } + + @Override + public CharSequence getText(CharSequence text) { + return null; + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/InvalidatingIterator.java b/document/src/main/java/com/yahoo/document/annotation/InvalidatingIterator.java new file mode 100644 index 00000000000..3542ee247a2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/InvalidatingIterator.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import java.util.ListIterator; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +class InvalidatingIterator implements ListIterator<SpanNode> { + private SpanList owner; + private ListIterator<SpanNode> base; + private SpanNode returnedFromNext = null; + + InvalidatingIterator(SpanList owner, ListIterator<SpanNode> base) { + this.owner = owner; + this.base = base; + } + + @Override + public boolean hasNext() { + returnedFromNext = null; + return base.hasNext(); + } + + @Override + public SpanNode next() { + SpanNode retval = null; + try { + retval = base.next(); + } finally { + returnedFromNext = retval; + } + return returnedFromNext; + } + + @Override + public boolean hasPrevious() { + returnedFromNext = null; + return base.hasPrevious(); + } + + @Override + public SpanNode previous() { + returnedFromNext = null; + return base.previous(); + } + + @Override + public int nextIndex() { + returnedFromNext = null; + return base.nextIndex(); + } + + @Override + public int previousIndex() { + returnedFromNext = null; + return base.previousIndex(); + } + + @Override + public void remove() { + if (returnedFromNext != null) { + returnedFromNext.setInvalid(); + returnedFromNext.setParent(null); + owner.resetCachedFromAndTo(); + } + returnedFromNext = null; + base.remove(); + } + + @Override + public void set(SpanNode spanNode) { + if (returnedFromNext != null) { + returnedFromNext.setInvalid(); + returnedFromNext.setParent(null); + } + owner.resetCachedFromAndTo(); + returnedFromNext = null; + base.set(spanNode); + } + + @Override + public void add(SpanNode spanNode) { + returnedFromNext = null; + owner.resetCachedFromAndTo(); + base.add(spanNode); + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java new file mode 100644 index 00000000000..588bf5f2826 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import java.util.IdentityHashMap; +import java.util.Iterator; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +abstract class IteratingAnnotationContainer extends AnnotationContainer { + + @Override + Iterator<Annotation> iterator(SpanNode node) { + IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>(); + nodes.put(node, node); + return iterator(nodes); + } + + @Override + Iterator<Annotation> iteratorRecursive(SpanNode node) { + IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>(); + nodes.put(node, node); + { + Iterator<SpanNode> childrenIt = node.childIteratorRecursive(); + while (childrenIt.hasNext()) { + SpanNode child = childrenIt.next(); + nodes.put(child, child); + } + } + return iterator(nodes); + } + + abstract Iterator<Annotation> iterator(IdentityHashMap<SpanNode, SpanNode> nodes); +} diff --git a/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java new file mode 100644 index 00000000000..0f03b9b269b --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import java.util.Collection; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.NoSuchElementException; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ListAnnotationContainer extends IteratingAnnotationContainer { + private final List<Annotation> annotations = new LinkedList<Annotation>(); + + @Override + void annotateAll(Collection<Annotation> annotations) { + this.annotations.addAll(annotations); + } + + @Override + void annotate(Annotation a) { + annotations.add(a); + } + + @Override + Collection<Annotation> annotations() { + return annotations; + } + + @Override + Iterator<Annotation> iterator(IdentityHashMap<SpanNode, SpanNode> nodes) { + return new AnnotationIterator(annotations.listIterator(), nodes); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ListAnnotationContainer)) return false; + ListAnnotationContainer that = (ListAnnotationContainer) o; + if (!annotations.equals(that.annotations)) return false; + return true; + } + + @Override + public int hashCode() { + return annotations.hashCode(); + } + + private class AnnotationIterator implements Iterator<Annotation> { + private IdentityHashMap<SpanNode, SpanNode> nodes; + private PeekableListIterator<Annotation> base; + private boolean nextCalled = false; + + AnnotationIterator(ListIterator<Annotation> baseIt, IdentityHashMap<SpanNode, SpanNode> nodes) { + this.base = new PeekableListIterator<Annotation>(baseIt); + this.nodes = nodes; + } + + @Override + public boolean hasNext() { + nextCalled = false; + while (base.hasNext() && !nodes.containsKey(base.peek().getSpanNode())) { + base.next(); + } + //now either, base has no next, or next is the correct node + if (base.hasNext()) { + return true; + } + return false; + } + + @Override + public Annotation next() { + if (hasNext()) { + nextCalled = true; + return base.next(); + } else { + throw new NoSuchElementException(); + } + } + + @Override + public void remove() { + if (!nextCalled) { + throw new IllegalStateException(); + } + base.remove(); + nextCalled = false; + } + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/PeekableListIterator.java b/document/src/main/java/com/yahoo/document/annotation/PeekableListIterator.java new file mode 100644 index 00000000000..53a8a1b803e --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/PeekableListIterator.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import java.util.ListIterator; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +class PeekableListIterator<E> implements ListIterator<E> { + private E next; + private ListIterator<E> base; + boolean traversed = false; + private int position = -1; + + PeekableListIterator(ListIterator<E> base) { + this.base = base; + this.traversed = false; + } + + @Override + public boolean hasNext() { + return next != null || base.hasNext(); + } + + @Override + public E next() { + if (next == null) { + E n = base.next(); + position++; + return n; + } + E retval = next; + next = null; + position++; + return retval; + } + + @Override + public boolean hasPrevious() { + throw new UnsupportedOperationException(); + } + + @Override + public E previous() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextIndex() { + throw new UnsupportedOperationException(); + } + + @Override + public int previousIndex() { + throw new UnsupportedOperationException(); + } + + @Override + public void remove() { + if (position < 0) { + //we have not next'ed the iterator, cannot do this: + throw new IllegalStateException("Cannot remove() before next()"); + } + if (next != null) { + //we have already gone one step ahead. must back up two positions and then remove: + base.previous(); + base.previous(); + base.remove(); + } else { + base.remove(); + } + next = null; + } + + @Override + public void set(E e) { + if (position < 0) { + //we have not next'ed the iterator, cannot do this: + throw new IllegalStateException("Cannot set() before next()"); + } + if (next != null) { + //we have already gone one step ahead. must back up two positions and then remove: + base.previous(); + base.previous(); + } + base.set(e); + next = null; + + } + + @Override + public void add(E e) { + if (next != null) { + //we have already gone one step ahead. must back up one position and then add: + base.previous(); + } + base.add(e); + next = null; + } + + public E peek() { + if (next == null && base.hasNext()) { + next = base.next(); + } + return next; + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java b/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java new file mode 100644 index 00000000000..ceecfcc2917 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/RecursiveNodeIterator.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import java.util.ListIterator; +import java.util.NoSuchElementException; +import java.util.Stack; + +/** + * ListIterator implementation which performs a depth-first traversal of SpanNodes. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +class RecursiveNodeIterator implements ListIterator<SpanNode> { + protected Stack<PeekableListIterator<SpanNode>> stack = new Stack<PeekableListIterator<SpanNode>>(); + protected ListIterator<SpanNode> iteratorFromLastCallToNext = null; + + RecursiveNodeIterator(ListIterator<SpanNode> it) { + stack.push(new PeekableListIterator<SpanNode>(it)); + } + + protected RecursiveNodeIterator() { + } + + @Override + public boolean hasNext() { + if (stack.isEmpty()) { + return false; + } + PeekableListIterator<SpanNode> iterator = stack.peek(); + if (!iterator.hasNext()) { + stack.pop(); + return hasNext(); + } + + + SpanNode node = iterator.peek(); + + if (!iterator.traversed) { + //we set the traversed flag on our way down + iterator.traversed = true; + stack.push(new PeekableListIterator<SpanNode>(node.childIterator())); + return hasNext(); + } + + return true; + } + + @Override + public SpanNode next() { + if (stack.isEmpty() || !hasNext()) { + iteratorFromLastCallToNext = null; + throw new NoSuchElementException("No next element available."); + } + stack.peek().traversed = false; + iteratorFromLastCallToNext = stack.peek(); + return stack.peek().next(); + } + + @Override + public boolean hasPrevious() { + throw new UnsupportedOperationException(); + } + + @Override + public SpanNode previous() { + throw new UnsupportedOperationException(); + } + + @Override + public int nextIndex() { + throw new UnsupportedOperationException(); + } + + @Override + public int previousIndex() { + throw new UnsupportedOperationException(); + } + + @Override + public void remove() { + if (stack.isEmpty()) { + throw new IllegalStateException(); + } + if (iteratorFromLastCallToNext != null) { + iteratorFromLastCallToNext.remove(); + } else { + throw new IllegalStateException(); + } + } + + @Override + public void set(SpanNode spanNode) { + if (stack.isEmpty()) { + throw new IllegalStateException(); + } + if (iteratorFromLastCallToNext != null) { + iteratorFromLastCallToNext.set(spanNode); + } else { + throw new IllegalStateException(); + } + } + + @Override + public void add(SpanNode spanNode) { + throw new UnsupportedOperationException(); + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/SerialIterator.java b/document/src/main/java/com/yahoo/document/annotation/SerialIterator.java new file mode 100644 index 00000000000..d86d90dc0d0 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/SerialIterator.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + + +import java.util.List; +import java.util.ListIterator; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SerialIterator extends RecursiveNodeIterator { + SerialIterator(List<ListIterator<SpanNode>> iterators) { + //the first iterator must be on top of the stack: + for (int i = iterators.size() - 1; i > -1; i--) { + stack.push(new PeekableListIterator<SpanNode>(iterators.get(i))); + } + } + + @Override + public boolean hasNext() { + if (stack.isEmpty()) { + return false; + } + PeekableListIterator<SpanNode> iterator = stack.peek(); + if (!iterator.hasNext()) { + stack.pop(); + return hasNext(); + } + return true; + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/Span.java b/document/src/main/java/com/yahoo/document/annotation/Span.java new file mode 100644 index 00000000000..87bd568b94a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/Span.java @@ -0,0 +1,180 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.yahoo.document.serialization.SpanNodeReader; + +import java.util.ListIterator; +import java.util.NoSuchElementException; + +/** + * This class represents a range of characters from a string. This is the leaf node + * in a Span tree. Its boundaries are defined as inclusive-from and exclusive-to. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class Span extends SpanNode { + public static final byte ID = 1; + private int from; + private int length; + + /** + * This will construct a valid span or throw {@link IllegalArgumentException} + * if the span is invalid. + * + * @param from Start of the span. Must be >= 0. + * @param length of the span. Must be >= 0. + * @throws IllegalArgumentException if illegal span + */ + public Span(int from, int length) { + setFrom(from); + setLength(length); + } + + /** + * Creates an empty Span, used mainly for deserialization. + * + * @param reader the reader that must populate this Span instance + */ + public Span(SpanNodeReader reader) { + reader.read(this); + } + + /** + * WARNING! Only to be used by deserializers! Creates an empty Span instance. + */ + public Span() { + } + + /** + * Copies the given Span into a new Span instance. + * + * @param spanToCopy the Span to copy. + */ + public Span(Span spanToCopy) { + this(spanToCopy.getFrom(), spanToCopy.getLength()); + } + + @Override + public final int getFrom() { + return from; + } + + /** + * NOTE: DO NOT USE. Should only be used by {@link com.yahoo.document.serialization.SpanNodeReader}. + * @param from the from value to set + */ + public void setFrom(int from) { + if (from < 0) { + throw new IllegalArgumentException("From cannot be < 0. (Was " + from + ")."); + } + this.from = from; + } + + @Override + public final int getTo() { + return from + length; + } + + @Override + public final int getLength() { + return length; + } + + /** + * NOTE: DO NOT USE. Should only be used by {@link com.yahoo.document.serialization.SpanNodeReader}. + * @param length the length value to set + */ + public void setLength(int length) { + if (length < 0) { + throw new IllegalArgumentException("Length cannot be < 0. (Was " + length + ")."); + } + this.length = length; + } + + public String toString() { + return new StringBuilder("span [").append(from).append(',').append(getTo()).append('>').toString(); + } + + @Override + public final CharSequence getText(CharSequence text) { + return text.subSequence(from, getTo()); + } + + /** + * Always returns true. + * + * @return always true. + */ + @Override + public boolean isLeafNode() { + return true; + } + + /** + * Returns a ListIterator that iterates over absolutely nothing. + * + * @return a ListIterator that iterates over absolutely nothing. + */ + @Override + public ListIterator<SpanNode> childIterator() { + return new EmptyIterator(); + } + + /** + * Returns a ListIterator that iterates over absolutely nothing. + * + * @return a ListIterator that iterates over absolutely nothing. + */ + @Override + public ListIterator<SpanNode> childIteratorRecursive() { + return childIterator(); + } + + private class EmptyIterator implements ListIterator<SpanNode> { + @Override + public boolean hasNext() { + return false; + } + + @Override + public SpanNode next() { + throw new NoSuchElementException("A Span has no children"); + } + + @Override + public boolean hasPrevious() { + return false; + } + + @Override + public SpanNode previous() { + throw new NoSuchElementException("A Span has no children"); + } + + @Override + public int nextIndex() { + return 0; + } + + @Override + public int previousIndex() { + return 0; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("A Span has no children"); + } + + @Override + public void set(SpanNode spanNode) { + throw new UnsupportedOperationException("A Span has no children"); + } + + @Override + public void add(SpanNode spanNode) { + throw new UnsupportedOperationException("A Span has no children"); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanList.java b/document/src/main/java/com/yahoo/document/annotation/SpanList.java new file mode 100644 index 00000000000..baa86bd7a6f --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/SpanList.java @@ -0,0 +1,418 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.yahoo.document.serialization.SpanNodeReader; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; + +/** + * A node in a Span tree that can have child nodes. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SpanList extends SpanNode { + public static final byte ID = 2; + private final List<SpanNode> children; + private int cachedFrom = Integer.MIN_VALUE; //triggers calculateFrom() + private int cachedTo = Integer.MIN_VALUE; //triggers calculateTo() + + /** Creates a new SpanList. */ + public SpanList() { + this.children = new LinkedList<SpanNode>(); + } + + public SpanList(SpanNodeReader reader) { + this(); + reader.read(this); + } + + protected SpanList(List<SpanNode> children) { + this.children = children; + } + + /** + * Deep-copies a SpanList. + * + * @param other the SpanList to copy. + */ + public SpanList(SpanList other) { + this.children = new LinkedList<SpanNode>(); + for (SpanNode otherNode : other.children) { + if (otherNode instanceof Span) { + children.add(new Span((Span) otherNode)); + } else if (otherNode instanceof AlternateSpanList) { + children.add(new AlternateSpanList((AlternateSpanList) otherNode)); + } else if (otherNode instanceof SpanList) { + children.add(new SpanList((SpanList) otherNode)); + } else if (otherNode instanceof DummySpanNode) { + children.add(otherNode); //shouldn't really happen + } else { + throw new IllegalStateException("Cannot create copy of " + otherNode + " with class " + + ((otherNode == null) ? "null" : otherNode.getClass())); + } + } + } + + void checkValidity(SpanNode node, List<SpanNode> childrenToCheck) { + if (!node.isValid()) { + throw new IllegalStateException("Cannot reuse SpanNode instance " + node + ", is INVALID."); + } + if (node.getParent() != null) { + if (node.getParent() != this) { + throw new IllegalStateException(node + " is already a child of " + node.getParent() + ", cannot be added to " + this); + } else if (node.getParent() == this && childrenToCheck.contains(node)) { + throw new IllegalStateException(node + " is already a child of " + this + ", cannot be added twice to the same parent node."); + } + } + } + + /** + * Adds a child node to this SpanList. + * + * @param node the node to add. + * @return this, for call chaining + * @throws IllegalStateException if SpanNode.isValid() returns false. + */ + public SpanList add(SpanNode node) { + checkValidity(node, children()); + node.setParent(this); + resetCachedFromAndTo(); + children().add(node); + return this; + } + + /** Create a span, add it to this list and return it */ + public Span span(int from, int length) { + Span span=new Span(from,length); + add(span); + return span; + } + + void setInvalid() { + //invalidate ourselves: + super.setInvalid(); + //invalidate all our children: + for (SpanNode node : children()) { + node.setInvalid(); + } + } + + /** + * Moves a child of this SpanList to another SpanList. + * + * @param node the node to move + * @param target the SpanList to add the node to + * @throws IllegalArgumentException if the given node is not a child of this SpanList + */ + public void move(SpanNode node, SpanList target) { + boolean removed = children().remove(node); + if (removed) { + //we found the node + node.setParent(null); + resetCachedFromAndTo(); + target.add(node); + } else { + throw new IllegalArgumentException("Node " + node + " is not a child of this SpanList, cannot move."); + } + } + + /** + * Moves a child of this SpanList to another SpanList. + * + * @param nodeNum the index of the node to move + * @param target the SpanList to add the node to + * @throws IndexOutOfBoundsException if the given index is out of range + */ + public void move(int nodeNum, SpanList target) { + SpanNode node = children().remove(nodeNum); + if (node != null) { + //we found the node + node.setParent(null); + resetCachedFromAndTo(); + target.add(node); + } + } + + /** + * Moves a child of this SpanList to another SpanList. + * + * @param node the node to move + * @param target the SpanList to add the node to + * @param targetSubTree the index of the subtree of the given AlternateSpanList to add the node to + * @throws IllegalArgumentException if the given node is not a child of this SpanList + * @throws IndexOutOfBoundsException if the target subtree index is out of range + */ + public void move(SpanNode node, AlternateSpanList target, int targetSubTree) { + if (targetSubTree < 0 || targetSubTree >= target.getNumSubTrees()) { + throw new IndexOutOfBoundsException(target + " has no subtree at index " + targetSubTree); + } + boolean removed = children().remove(node); + if (removed) { + //we found the node + node.setParent(null); + resetCachedFromAndTo(); + target.add(targetSubTree, node); + } else { + throw new IllegalArgumentException("Node " + node + " is not a child of this SpanList, cannot move."); + } + } + + /** + * Moves a child of this SpanList to another SpanList. + * + * @param nodeNum the index of the node to move + * @param target the SpanList to add the node to + * @param targetSubTree the index of the subtree of the given AlternateSpanList to add the node to + * @throws IndexOutOfBoundsException if the given index is out of range, or the target subtree index is out of range + */ + public void move(int nodeNum, AlternateSpanList target, int targetSubTree) { + if (targetSubTree < 0 || targetSubTree >= target.getNumSubTrees()) { + throw new IndexOutOfBoundsException(target + " has no subtree at index " + targetSubTree); + } + SpanNode node = children().remove(nodeNum); + if (node != null) { + //we found the node + node.setParent(null); + resetCachedFromAndTo(); + target.add(targetSubTree, node); + } + } + + + /** + * Removes and invalidates the given SpanNode from this. + * + * @param node the node to remove. + * @return this, for chaining. + */ + public SpanList remove(SpanNode node) { + boolean removed = children().remove(node); + if (removed) { + node.setParent(null); + resetCachedFromAndTo(); + node.setInvalid(); + } + return this; + } + + /** + * Removes and invalidates the SpanNode at the given index from this. + * + * @param i the index of the node to remove. + * @return this, for chaining. + */ + public SpanList remove(int i) { + SpanNode node = children().remove(i); + if (node != null) { + node.setParent(null); + node.setInvalid(); + } + return this; + } + + /** + * Returns a <strong>modifiable</strong> list of the immediate children of this SpanList. + * + * @return a <strong>modifiable</strong> list of the immediate children of this SpanList. + */ + protected List<SpanNode> children() { + return children; + } + + /** + * Returns the number of children this SpanList holds. + * + * @return the number of children this SpanList holds. + */ + public int numChildren() { + return children().size(); + } + + /** + * Traverses all immediate children of this SpanList. The ListIterator returned support all optional operations + * specified in the ListIterator interface. + * + * @return a ListIterator which traverses all immediate children of this SpanNode + * @see java.util.ListIterator + */ + @Override + public ListIterator<SpanNode> childIterator() { + return new InvalidatingIterator(this, children().listIterator()); + } + + /** + * Recursively traverses all children (not only leaf nodes) of this SpanList, in a depth-first fashion. + * The ListIterator only supports iteration forwards, and the optional operations that are implemented are + * remove() and set(). add() is not supported. + * + * @return a ListIterator which recursively traverses all children and their children etc. of this SpanList. + * @see java.util.ListIterator + */ + @Override + public ListIterator<SpanNode> childIteratorRecursive() { + return new InvalidatingIterator(this, new RecursiveNodeIterator(children().listIterator())); + } + + /** Removes and invalidates all references to child nodes. */ + public void clearChildren() { + for (SpanNode node : children()) { + node.setInvalid(); + node.setParent(null); + } + children().clear(); + resetCachedFromAndTo(); + } + + /** + * Sorts children by occurrence in the text covered. + * + * @see SpanNode#compareTo(SpanNode) + */ + public void sortChildren() { + Collections.sort(children()); + } + + /** + * Recursively sorts all children by occurrence in the text covered. + */ + public void sortChildrenRecursive() { + for (SpanNode node : children()) { + if (node instanceof SpanList) { + ((SpanList) node).sortChildrenRecursive(); + } + Collections.sort(children()); + } + } + + /** + * Always returns false, even if this node has no children. + * + * @return always false, even if this node has no children + */ + @Override + public boolean isLeafNode() { + return false; + } + + private void calculateFrom() { + int smallestFrom = Integer.MAX_VALUE; + for (SpanNode n : children()) { + final int thisFrom = n.getFrom(); + if (thisFrom != -1) { + smallestFrom = Math.min(thisFrom, smallestFrom); + } + } + if (smallestFrom == Integer.MAX_VALUE) { + //all children were empty SpanLists which returned -1 + smallestFrom = -1; + } + cachedFrom = smallestFrom; + } + + /** + * Returns the character index where this {@link SpanNode} starts (inclusive), i.e. the smallest {@link com.yahoo.document.annotation.SpanNode#getFrom()} of all children. + * + * @return the lowest getFrom() of all children, or -1 if this SpanList has no children. + */ + @Override + public int getFrom() { + if (children().isEmpty()) { + return -1; + } + if (cachedFrom == Integer.MIN_VALUE) { + calculateFrom(); + } + return cachedFrom; + } + + private void calculateTo() { + int greatestTo = Integer.MIN_VALUE; + for (SpanNode n : children()) { + greatestTo = Math.max(n.getTo(), greatestTo); + } + cachedTo = greatestTo; + } + + /** + * Returns the character index where this {@link SpanNode} ends (exclusive), i.e. the greatest {@link com.yahoo.document.annotation.SpanNode#getTo()} of all children. + * + * @return the greatest getTo() of all children, or -1 if this SpanList has no children. + */ + @Override + public int getTo() { + if (children().isEmpty()) { + return -1; + } + if (cachedTo == Integer.MIN_VALUE) { + calculateTo(); + } + return cachedTo; + } + + void resetCachedFromAndTo() { + cachedFrom = Integer.MIN_VALUE; + cachedTo = Integer.MIN_VALUE; + if (getParent() instanceof SpanList) { + ((SpanList) getParent()).resetCachedFromAndTo(); + } + } + + /** + * Returns the length of this span, i.e. getFrom() - getTo(). + * + * @return the length of this span + */ + @Override + public int getLength() { + return getTo() - getFrom(); + } + + /** + * Returns the text that is covered by this SpanNode. + * + * @param text the input text + * @return the text that is covered by this SpanNode. + */ + @Override + public CharSequence getText(CharSequence text) { + if (children().isEmpty()) { + return ""; + } + StringBuilder str = new StringBuilder(); + for (SpanNode node : children()) { + CharSequence childText = node.getText(text); + if (childText != null) { + str.append(node.getText(text)); + } + } + return str; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SpanList)) return false; + if (!super.equals(o)) return false; + + SpanList spanList = (SpanList) o; + + if (children() != null ? !children().equals(spanList.children()) : spanList.children() != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (children() != null ? children().hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "SpanList with " + children().size() + " children"; + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanNode.java b/document/src/main/java/com/yahoo/document/annotation/SpanNode.java new file mode 100644 index 00000000000..712bb7bf5c5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/SpanNode.java @@ -0,0 +1,320 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; + +import java.util.ListIterator; + +/** + * Base class of nodes in a Span tree. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public abstract class SpanNode implements Comparable<SpanNode>, SpanNodeParent { + + private boolean valid = true; + /** + * This scratch id is used to avoid using IdentityHashMaps as they are very costly. + */ + private int scratchId = -1; + private SpanNodeParent parent; + + protected SpanNode() { + } + + /** + * Returns whether this node is valid or not. When a child node from a SpanList, the child + * is marked as invalid, and the reference to it is removed from the parent SpanList. However, + * Annotations in the global list kept in SpanTree may still have references to the removed SpanNode. + * Removing these references is costly, and is only done when calling {@link com.yahoo.document.annotation.SpanTree#cleanup()}. + * + * @return true if this node is valid, false otherwise. + */ + public boolean isValid() { + return valid; + } + + void setInvalid() { + valid = false; + } + + public void setScratchId(int id) { + scratchId = id; + } + + public int getScratchId() { + return scratchId; + } + /** + * Returns the parent node of this SpanNode, if any. + * + * @return the parent node, or null if this is not yet added to a parent SpanList + */ + public SpanNodeParent getParent() { + return parent; + } + + void setParent(SpanNodeParent parent) { + this.parent = parent; + } + + /** + * Returns the SpanTree that this node belongs to, if any. + * + * @return the SpanTree that this node belongs to, or null if it is not yet added to a SpanTree. + */ + @Override + public SpanTree getSpanTree() { + if (parent == null) { + return null; + } + return parent.getSpanTree(); + } + + /** Returns the SpanTree this belongs to and throws a nice NullPointerException if none */ + private SpanTree getNonNullSpanTree() { + SpanTree spanTree=getSpanTree(); + if (spanTree==null) + throw new NullPointerException(this + " is not attached to a SpanTree through its parent yet"); + return spanTree; + } + + + /** + * Convenience method for adding an annotation to this span, same as + * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,Annotation) spanTree.annotate(this,annotation)}</code> + * + * @param annotation the annotation to add + * @return this for chaining + * @throws NullPointerException if this span is not attached to a tree + */ + public SpanNode annotate(Annotation annotation) { + getNonNullSpanTree().annotate(this, annotation); + return this; + } + + /** + * Convenience method for adding an annotation to this span, same as + * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,AnnotationType,FieldValue) spanTree.annotate(this,type,value)}</code> + * + * @param type the type of the annotation to add + * @param value the value of the annotation to add + * @return this for chaining + * @throws NullPointerException if this span is not attached to a tree + */ + public SpanNode annotate(AnnotationType type,FieldValue value) { + getNonNullSpanTree().annotate(this,type,value); + return this; + } + + /** + * Convenience method for adding an annotation to this span, same as + * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,AnnotationType,FieldValue) spanTree.annotate(this,type,new StringFieldValue(value))}</code> + * + * @param type the type of the annotation to add + * @param value the string value of the annotation to add + * @return this for chaining + * @throws NullPointerException if this span is not attached to a tree + */ + public SpanNode annotate(AnnotationType type,String value) { + getNonNullSpanTree().annotate(this, type, new StringFieldValue(value)); + return this; + } + + /** + * Convenience method for adding an annotation to this span, same as + * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,AnnotationType,FieldValue) spanTree.annotate(this,type,new IntegerFieldValue(value))}</code> + * + * @param type the type of the annotation to add + * @param value the integer value of the annotation to add + * @return this for chaining + * @throws NullPointerException if this span is not attached to a tree + */ + public SpanNode annotate(AnnotationType type,Integer value) { + getNonNullSpanTree().annotate(this, type, new IntegerFieldValue(value)); + return this; + } + + /** + * Convenience method for adding an annotation with no value to this span, same as + * <code>getSpanTree().{@link SpanTree#annotate(SpanNode,AnnotationType) spanTree.annotate(this,type)}</code> + * + * @param type the type of the annotation to add + * @return this for chaining + * @throws NullPointerException if this span is not attached to a tree + */ + public SpanNode annotate(AnnotationType type) { + getNonNullSpanTree().annotate(this,type); + return this; + } + + /** + * Returns the StringFieldValue that this node belongs to, if any. + * + * @return the StringFieldValue that this node belongs to, if any, otherwise null. + */ + @Override + public StringFieldValue getStringFieldValue() { + if (parent == null) { + return null; + } + return parent.getStringFieldValue(); + } + + /** + * Returns true if this node is a leaf node in the tree. + * + * @return true if this node is a leaf node in the tree. + */ + public abstract boolean isLeafNode(); + + /** + * Traverses all immediate children of this SpanNode. + * + * @return a ListIterator which traverses all immediate children of this SpanNode + */ + public abstract ListIterator<SpanNode> childIterator(); + + /** + * Recursively traverses all possible children (not only leaf nodes) of this SpanNode, in a depth-first fashion. + * + * @return a ListIterator which recursively traverses all children and their children etc. of this SpanNode. + */ + public abstract ListIterator<SpanNode> childIteratorRecursive(); + + /** + * Returns the character index where this SpanNode starts (inclusive). + * + * @return the character index where this SpanNode starts (inclusive). + */ + public abstract int getFrom(); + + /** + * Returns the character index where this SpanNode ends (exclusive). + * + * @return the character index where this SpanNode ends (exclusive). + */ + public abstract int getTo(); + + /** + * Returns the length of this span, i.e. getFrom() - getTo(). + * + * @return the length of this span + */ + public abstract int getLength(); + + /** + * Returns the text that is covered by this SpanNode. + * + * @param text the input text + * @return the text that is covered by this SpanNode. + */ + public abstract CharSequence getText(CharSequence text); + + /** + * Checks if the text covered by this span overlaps with the text covered by another span. + * + * @param o the other SpanNode to check + * @return true if spans are overlapping, false otherwise + */ + public boolean overlaps(SpanNode o) { + int from = getFrom(); + int otherFrom = o.getFrom(); + int to = getTo(); + int otherTo = o.getTo(); + + //is other from within our range, or vice versa? + if ((otherFrom >= from && otherFrom < to) + || (from >= otherFrom && from < otherTo)) { + return true; + } + + //is other to within our range, or vice versa? + if ((otherTo > from && otherTo <= to) + || (to > otherFrom && to <= otherTo)) { + return true; + } + return false; + } + + /** + * Checks if the text covered by another span is within the text covered by this span. + * + * @param o the other SpanNode to check. + * @return true if the text covered by another span is within the text covered by this span, false otherwise. + */ + public boolean contains(SpanNode o) { + int from = getFrom(); + int otherFrom = o.getFrom(); + int to = getTo(); + int otherTo = o.getTo(); + + if (otherFrom >= from && otherTo <= to) { + //other span node is within our range: + return true; + } + return false; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SpanNode)) return false; + + SpanNode spanNode = (SpanNode) o; + + if (getFrom() != spanNode.getFrom()) return false; + if (getTo() != spanNode.getTo()) return false; + + return true; + } + + @Override + public int hashCode() { + int result = getFrom(); + result = 31 * result + getTo(); + return result; + } + + /** + * Compares two SpanNodes. Note: this class has a natural ordering that <strong>might be</strong> inconsistent with equals. + * <p> + * First, getFrom() is compared, and -1 or 1 is return if our getFrom() is smaller or greater that o.getFrom(), respectively. + * If and only if getFrom() is equal, getTo() is compared, and -1 or 1 is return if our getTo() is smaller or greater that o.getTo(), respectively. + * In all other cases, the two SpanNodes are equal both for getFrom() and getTo(), and 0 is returned. + * <p> + * Note that if a subclass has overridden equals(), which is very likely, but has not overridden compareTo(), then that subclass + * will have a natural ordering that is inconsistent with equals. + * + * @param o the SpanNode to compare to + * @return a negative integer, zero, or a positive integer as this object is less than, equal to, or greater than the specified object + */ + @Override + public int compareTo(SpanNode o) { + int from = getFrom(); + int otherFrom = o.getFrom(); + + if (from < otherFrom) { + return -1; + } + if (from > otherFrom) { + return 1; + } + + //so from is equal. Check to: + int to = getTo(); + int otherTo = o.getTo(); + + if (to < otherTo) { + return -1; + } + if (to > otherTo) { + return 1; + } + + //both from and to are equal + return 0; + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java new file mode 100644 index 00000000000..8f10d7c0140 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import org.apache.commons.collections.map.MultiValueMap; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.List; + +/** + * TODO: Should this be removed? + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +class SpanNode2AnnotationContainer extends AnnotationContainer { + private final MultiValueMap spanNode2Annotation = MultiValueMap.decorate(new IdentityHashMap()); + + @Override + void annotateAll(Collection<Annotation> annotations) { + for (Annotation a : annotations) { + annotate(a); + } + } + + @Override + void annotate(Annotation a) { + if (a.getSpanNode() == null) { + spanNode2Annotation.put(DummySpanNode.INSTANCE, a); + } else { + spanNode2Annotation.put(a.getSpanNode(), a); + } + } + + @Override + @SuppressWarnings("unchecked") + Collection<Annotation> annotations() { + return spanNode2Annotation.values(); + } + + @Override + @SuppressWarnings("unchecked") + Iterator<Annotation> iterator(SpanNode node) { + Collection<Annotation> annotationsForNode = spanNode2Annotation.getCollection(node); + if (annotationsForNode == null) { + return Collections.<Annotation>emptyList().iterator(); + } + return annotationsForNode.iterator(); } + + @Override + @SuppressWarnings("unchecked") + Iterator<Annotation> iteratorRecursive(SpanNode node) { + IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>(); + nodes.put(node, node); + { + Iterator<SpanNode> childrenIt = node.childIteratorRecursive(); + while (childrenIt.hasNext()) { + SpanNode child = childrenIt.next(); + nodes.put(child, child); + } + } + List<Collection<Annotation>> annotationLists = new ArrayList<Collection<Annotation>>(nodes.size()); + for (SpanNode includedNode : nodes.keySet()) { + Collection<Annotation> includedAnnotations = spanNode2Annotation.getCollection(includedNode); + if (includedAnnotations != null) { + annotationLists.add(includedAnnotations); + } + } + return new AnnotationCollectionIterator(annotationLists); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SpanNode2AnnotationContainer)) return false; + SpanNode2AnnotationContainer that = (SpanNode2AnnotationContainer) o; + if (!spanNode2Annotation.equals(that.spanNode2Annotation)) return false; + return true; + } + + @Override + public int hashCode() { + return spanNode2Annotation.hashCode(); + } + + private class AnnotationCollectionIterator implements Iterator<Annotation> { + private final List<Collection<Annotation>> annotationLists; + private Iterator<Annotation> currentIterator; + private boolean nextCalled = false; + + AnnotationCollectionIterator(List<Collection<Annotation>> annotationLists) { + this.annotationLists = annotationLists; + if (annotationLists.isEmpty()) { + currentIterator = Collections.<Annotation>emptyList().iterator(); + } else { + currentIterator = annotationLists.remove(0).iterator(); + } + } + + @Override + public boolean hasNext() { + nextCalled = false; + if (currentIterator.hasNext()) { + return true; + } + if (annotationLists.isEmpty()) { + return false; + } + currentIterator = annotationLists.remove(0).iterator(); + return hasNext(); + } + + @Override + public Annotation next() { + if (hasNext()) { + nextCalled = true; + return currentIterator.next(); + } + return null; + } + + @Override + public void remove() { + if (nextCalled) { + currentIterator.remove(); + } else { + throw new IllegalStateException(); + } + } + } + +} diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java b/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java new file mode 100644 index 00000000000..8618685d77a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/SpanNodeParent.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * An interface to be implemented by classes that can be parents of SpanNodes. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @see SpanNode#getParent() + */ +public interface SpanNodeParent { + /** + * Returns the SpanTree of this, if any. + * + * @return the SpanTree of this, if it belongs to a SpanTree, otherwise null. + */ + public SpanTree getSpanTree(); + + /** + * Returns the StringFieldValue that this node belongs to, if any. + * + * @return the StringFieldValue that this node belongs to, if any, otherwise null. + */ + public StringFieldValue getStringFieldValue(); +} diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java new file mode 100644 index 00000000000..7385461504d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java @@ -0,0 +1,699 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +import com.google.common.collect.ImmutableList; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructuredDataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; +import org.apache.commons.collections.CollectionUtils; + +import java.util.*; + +/** + * A SpanTree holds a root node of a tree of SpanNodes, and a List of Annotations pointing to these nodes + * or each other. It also has a name. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @see com.yahoo.document.annotation.SpanNode + * @see com.yahoo.document.annotation.Annotation + */ +public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparable<SpanTree> { + + private String name; + private SpanNode root; + private AnnotationContainer annotations = new ListAnnotationContainer(); + private StringFieldValue stringFieldValue; + + /** + * WARNING! Only to be used by deserializers! Creates an empty SpanTree instance. + */ + public SpanTree() { + } + + /** + * Creates a new SpanTree with a given root node. + * + * @param name the name of the span tree + * @param root the root node of the span tree + * @throws IllegalStateException if the root node is invalid + */ + public SpanTree(String name, SpanNode root) { + this.name = name; + setRoot(root); + } + + /** + * Creates a new SpanTree with the given name and an empty SpanList as its root node. + * + * @param name the name of the span tree + */ + public SpanTree(String name) { + this.name = name; + setRoot(new SpanList()); + } + + @SuppressWarnings("unchecked") + public SpanTree(SpanTree otherToCopy) { + name = otherToCopy.name; + setRoot(copySpan(otherToCopy.root)); + List<Annotation> annotationsToCopy = new ArrayList<Annotation>(otherToCopy.getAnnotations()); + List<Annotation> newAnnotations = new ArrayList<Annotation>(annotationsToCopy.size()); + + for (Annotation otherAnnotationToCopy : annotationsToCopy) { + newAnnotations.add(new Annotation(otherAnnotationToCopy)); + } + + IdentityHashMap<SpanNode, Integer> originalSpanNodes = getSpanNodes(otherToCopy); + List<SpanNode> copySpanNodes = getSpanNodes(); + + for (int i = 0; i < annotationsToCopy.size(); i++) { + Annotation originalAnnotation = annotationsToCopy.get(i); + if (!originalAnnotation.isSpanNodeValid()) { //returns false also if spanNode is null! + continue; + } + Integer indexOfOriginalSpanNode = originalSpanNodes.get(originalAnnotation.getSpanNode()); + if (indexOfOriginalSpanNode == null) { + throw new IllegalStateException("Could not clone tree, SpanNode of " + originalAnnotation + " not found."); + } + newAnnotations.get(i).setSpanNode(copySpanNodes.get(indexOfOriginalSpanNode)); + } + + IdentityHashMap<Annotation, Integer> originalAnnotations = getAnnotations(annotationsToCopy); + + for (Annotation a : newAnnotations) { + if (!a.hasFieldValue()) { + continue; + } + setCorrectAnnotationReference(a.getFieldValue(), originalAnnotations, newAnnotations); + } + + for (Annotation a : newAnnotations) { + annotate(a); + } + for (IndexKey key : otherToCopy.getCurrentIndexes()) { + createIndex(key); + } + } + + private void setCorrectAnnotationReference(FieldValue value, IdentityHashMap<Annotation, Integer> originalAnnotations, List<Annotation> newAnnotations) { + if (value == null) { + return; + } + + if (value.getDataType() instanceof AnnotationReferenceDataType) { + AnnotationReference ref = (AnnotationReference) value; + if (ref.getReference() == null) { + return; + } + Integer referenceIndex = originalAnnotations.get(ref.getReference()); + if (referenceIndex == null) { + throw new IllegalStateException("Cannot find Annotation pointed to by " + ref); + } + try { + Annotation newReference = newAnnotations.get(referenceIndex); + ref.setReference(newReference); + } catch (IndexOutOfBoundsException ioobe) { + throw new IllegalStateException("Cannot find Annotation pointed to by " + ref, ioobe); + } + } else if (value.getDataType() instanceof StructuredDataType) { + setCorrectAnnotationReference((StructuredFieldValue) value, originalAnnotations, newAnnotations); + } else if (value.getDataType() instanceof CollectionDataType) { + setCorrectAnnotationReference((CollectionFieldValue) value, originalAnnotations, newAnnotations); + } else if (value.getDataType() instanceof MapDataType) { + setCorrectAnnotationReference((MapFieldValue) value, originalAnnotations, newAnnotations); + } + } + + private void setCorrectAnnotationReference(StructuredFieldValue struct, IdentityHashMap<Annotation, Integer> originalAnnotations, List<Annotation> newAnnotations) { + for (Field f : struct.getDataType().getFields()) { + setCorrectAnnotationReference(struct.getFieldValue(f), originalAnnotations, newAnnotations); + } + } + + private void setCorrectAnnotationReference(CollectionFieldValue collection, IdentityHashMap<Annotation, Integer> originalAnnotations, List<Annotation> newAnnotations) { + Iterator it = collection.fieldValueIterator(); + while (it.hasNext()) { + setCorrectAnnotationReference((FieldValue) it.next(), originalAnnotations, newAnnotations); + } + } + + private void setCorrectAnnotationReference(MapFieldValue map, IdentityHashMap<Annotation, Integer> originalAnnotations, List<Annotation> newAnnotations) { + for (Object o : map.values()) { + setCorrectAnnotationReference((FieldValue) o, originalAnnotations, newAnnotations); + } + } + + private IdentityHashMap<Annotation, Integer> getAnnotations(List<Annotation> annotationsToCopy) { + IdentityHashMap<Annotation, Integer> map = new IdentityHashMap<Annotation, Integer>(); + for (int i = 0; i < annotationsToCopy.size(); i++) { + map.put(annotationsToCopy.get(i), i); + } + return map; + } + + + private List<SpanNode> getSpanNodes() { + ArrayList<SpanNode> nodes = new ArrayList<SpanNode>(); + nodes.add(root); + Iterator<SpanNode> it = root.childIteratorRecursive(); + while (it.hasNext()) { + nodes.add(it.next()); + } + return nodes; + } + + private static IdentityHashMap<SpanNode, Integer> getSpanNodes(SpanTree otherToCopy) { + IdentityHashMap<SpanNode, Integer> map = new IdentityHashMap<SpanNode, Integer>(); + int spanNodeCounter = 0; + map.put(otherToCopy.getRoot(), spanNodeCounter++); + Iterator<SpanNode> it = otherToCopy.getRoot().childIteratorRecursive(); + while (it.hasNext()) { + map.put(it.next(), spanNodeCounter++); + } + return map; + } + + private SpanNode copySpan(SpanNode spanTree) { + if (spanTree instanceof Span) { + return new Span((Span) spanTree); + } else if (spanTree instanceof AlternateSpanList) { + return new AlternateSpanList((AlternateSpanList) spanTree); + } else if (spanTree instanceof SpanList) { + return new SpanList((SpanList) spanTree); + } else if (spanTree instanceof DummySpanNode) { + return spanTree; //shouldn't really happen + } else { + throw new IllegalStateException("Cannot create copy of " + spanTree + " with class " + + ((spanTree == null) ? "null" : spanTree.getClass())); + } + } + + /** + * WARNING! Only to be used by deserializers! Sets the name of this SpanTree instance. + * + * @param name the name to set for this SpanTree instance. + */ + public void setName(String name) { + this.name = name; + } + + /** + * WARNING! Only to be used by deserializers! Sets the root of this SpanTree instance. + * + * @param root the root to set for this SpanTree instance. + */ + public void setRoot(SpanNode root) { + if (!root.isValid()) { + throw new IllegalStateException("Cannot use invalid node " + root + " as root node."); + } + if (root.getParent() != null) { + if (root.getParent() != this) { + throw new IllegalStateException(root + " is already a child of " + root.getParent() + ", cannot be root of " + this); + } + } + this.root = root; + root.setParent(this); + } + + /** + * Returns the name of this span tree. + * @return the name of this span tree. + */ + public String getName() { + return name; + } + + /** + * Returns the root node of this span tree. + * @return the root node of this span tree. + */ + public SpanNode getRoot() { + return root; + } + + /** + * Convenience shorthand for <code>(SpanList)getRoot()</code>. + * This must of course only be used when it is known that the root in this tree actually is a SpanList. + */ + public SpanList spanList() { + return (SpanList)root; + } + + /** + * Ensures consistency of the tree in case SpanNodes have been removed, and there are still + * Annotations pointing to them. This method has a maximum upper bound of O(3nm), where n is the + * total number of Annotations, and m is the number of SpanNodes that had been removed from the tree. + * The lower bound is Omega(n), if no SpanNodes had been removed from the tree. + */ + @SuppressWarnings("unchecked") + public void cleanup() { + Map<Annotation, Annotation> removedAnnotations = removeAnnotationsThatPointToInvalidSpanNodes(); + + //here: + //iterate through all annotations; + //if any of those have ONLY an annotationreference as its value, + // - null reference + // - remove value from annotation + // - remove annotation and add it to removedAnnotations map + if (!removedAnnotations.isEmpty()) { + Iterator<Annotation> annotationIt = iterator(); + while (annotationIt.hasNext()) { + Annotation a = annotationIt.next(); + if (!a.hasFieldValue()) { + continue; + } + FieldValue value = a.getFieldValue(); + + if (value instanceof AnnotationReference) { + //the annotation "a" has a reference + AnnotationReference ref = (AnnotationReference) value; + if (removedAnnotations.get(ref.getReference()) != null) { + //this reference refers to a dead annotation + ref.setReference(null); + a.setFieldValue(null); + if (!a.isSpanNodeValid()) { + //this annotation has no span node, delete it + annotationIt.remove(); + removedAnnotations.put(a, a); + } + } + } + } + } + + //there may still be references to removed annotations, + //inside maps, weighted sets, structs, etc. + //if any of those have such references, + // - null reference + // - remove annotationref from struct, map, etc. + // - apart from this, keep struct, map etc. and annotation + if (!removedAnnotations.isEmpty()) { + for (Annotation a : this) { + if (!a.hasFieldValue()) { + continue; + } + removeObsoleteReferencesFromFieldValue(a.getFieldValue(), removedAnnotations, true); + } + } + //was any annotations removed from the global list? do we still have references to those annotations + //that have been removed? if so, remove the references + removeAnnotationReferencesThatPointToRemovedAnnotations(); + } + + private boolean removeObsoleteReferencesFromFieldValue(FieldValue value, Map<Annotation, Annotation> selectedAnnotations, boolean removeIfPresent) { + if (value == null) { + return false; + } + + if (value.getDataType() instanceof AnnotationReferenceDataType) { + AnnotationReference ref = (AnnotationReference) value; + if (removeIfPresent) { + if (selectedAnnotations.containsValue(ref.getReference())) { + //this reference refers to a dead annotation + ref.setReference(null); + return true; + } + } else { + if (!selectedAnnotations.containsValue(ref.getReference())) { + //this reference refers to a dead annotation + ref.setReference(null); + return true; + } + } + } else if (value.getDataType() instanceof StructuredDataType) { + removeObsoleteReferencesFromStructuredType((StructuredFieldValue) value, selectedAnnotations, removeIfPresent); + } else if (value.getDataType() instanceof CollectionDataType) { + removeObsoleteReferencesFromCollectionType((CollectionFieldValue) value, selectedAnnotations, removeIfPresent); + } else if (value.getDataType() instanceof MapDataType) { + removeObsoleteReferencesFromMapType((MapFieldValue) value, selectedAnnotations, removeIfPresent); + } + return false; + } + + private boolean removeObsoleteReferencesFromStructuredType(StructuredFieldValue struct, Map<Annotation, Annotation> selectedAnnotations, boolean removeIfPresent) { + for (Field f : struct.getDataType().getFields()) { + FieldValue fValue = struct.getFieldValue(f); + if (removeObsoleteReferencesFromFieldValue(fValue, selectedAnnotations, removeIfPresent)) { + struct.removeFieldValue(f); + } + } + return false; + } + + private boolean removeObsoleteReferencesFromCollectionType(CollectionFieldValue collection, Map<Annotation, Annotation> selectedAnnotations, boolean removeIfPresent) { + Iterator it = collection.fieldValueIterator(); + while (it.hasNext()) { + FieldValue fValue = (FieldValue) it.next(); + if (removeObsoleteReferencesFromFieldValue(fValue, selectedAnnotations, removeIfPresent)) { + it.remove(); + } + } + return false; + } + + private boolean removeObsoleteReferencesFromMapType(MapFieldValue map, Map<Annotation, Annotation> selectedAnnotations, boolean removeIfPresent) { + Iterator valueIt = map.values().iterator(); + while (valueIt.hasNext()) { + FieldValue fValue = (FieldValue) valueIt.next(); + if (removeObsoleteReferencesFromFieldValue(fValue, selectedAnnotations, removeIfPresent)) { + valueIt.remove(); + } + } + return false; + } + + @SuppressWarnings("unchecked") + private Map<Annotation, Annotation> removeAnnotationsThatPointToInvalidSpanNodes() { + Map<Annotation, Annotation> removedAnnotations = new IdentityHashMap<Annotation, Annotation>(); + + Iterator<Annotation> annotationIt = iterator(); + while (annotationIt.hasNext()) { + Annotation a = annotationIt.next(); + if (a.hasSpanNode() && !a.isSpanNodeValid()) { + a.setSpanNode(null); + a.setFieldValue(null); + removedAnnotations.put(a, a); + annotationIt.remove(); + } + } + return removedAnnotations; + } + + private boolean hasAnyFieldValues() { + for (Annotation a : this) { + if (a.hasFieldValue()) { + return true; + } + } + return false; + } + private void removeAnnotationReferencesThatPointToRemovedAnnotations() { + if (hasAnyFieldValues()) { + Map<Annotation, Annotation> annotationsStillPresent = new IdentityHashMap<Annotation, Annotation>(); + for (Annotation a : this) { + annotationsStillPresent.put(a, a); + } + for (Annotation a : this) { + if (!a.hasFieldValue()) { + continue; + } + //do we have any references to annotations that are NOT in this global list?? + removeObsoleteReferencesFromFieldValue(a.getFieldValue(), annotationsStillPresent, false); + } + } + } + + private void annotateInternal(SpanNode node, Annotation annotation) { + annotations.annotate(annotation); + } + + @SuppressWarnings("unchecked") + private Collection<Annotation> getAnnotations() { + return annotations.annotations(); + } + + /** + * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when + * adding an Annotation that uses an AnnotationReference, and does not annotate a SpanNode. + * + * @param a the Annotation to add + * @return this, for chaining + * @see com.yahoo.document.annotation.Annotation + * @see com.yahoo.document.annotation.AnnotationReference + * @see com.yahoo.document.annotation.AnnotationReferenceDataType + */ + public SpanTree annotate(Annotation a) { + if (a.getSpanNode() == null) { + annotateInternal(DummySpanNode.INSTANCE, a); + } else { + annotateInternal(a.getSpanNode(), a); + } + return this; + } + + /** + * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when + * adding an Annotation that shall annotate a SpanNode. Upon return, Annotation.getSpanNode() + * returns the given node. + * + * @param node the node to annotate + * @param annotation the Annotation to add + * @return this, for chaining + * @see com.yahoo.document.annotation.Annotation + */ + public SpanTree annotate(SpanNode node, Annotation annotation) { + annotation.setSpanNode(node); + return annotate(annotation); + } + + /** + * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when + * adding an Annotation that shall annotate a SpanNode. Upon return, Annotation.getSpanNode() + * returns the given node. This one is unchecked and assumes that the SpanNode is valid and has + * already been attached to the Annotation. + * + * @param node the node to annotate + * @param annotation the Annotation to add + * @return this, for chaining + * @see com.yahoo.document.annotation.Annotation + */ + public final SpanTree annotateFast(SpanNode node, Annotation annotation) { + annotateInternal(node, annotation); + return this; + } + + /** + * Adds an Annotation. + * Convenience shorthand for <code>annotate(node,new Annotation(type,value)</code> + * + * @param node the node to annotate + * @param type the type of the Annotation to add + * @param value the value of the Annotation to add + * @return this, for chaining + * @see com.yahoo.document.annotation.Annotation + */ + public SpanTree annotate(SpanNode node, AnnotationType type,FieldValue value) { + return annotate(node, new Annotation(type, value)); + } + + /** + * Creates an Annotation based on the given AnnotationType, and adds it to the internal list of + * annotations for this SpanTree (convenience method). Use this when + * adding an Annotation (that does not have a FieldValue) that shall annotate a SpanNode. + * Upon return, Annotation.getSpanNode() + * returns the given node. + * + * @param node the node to annotate + * @param type the AnnotationType to create an Annotation from + * @return this, for chaining + * @see com.yahoo.document.annotation.Annotation + * @see com.yahoo.document.annotation.AnnotationType + */ + public SpanTree annotate(SpanNode node, AnnotationType type) { + Annotation a = new Annotation(type); + return annotate(node, a); + } + + /** + * Removes an Annotation from the internal list of annotations. + * + * @param a the annotation to remove + * @return true if the Annotation was successfully removed, false otherwise + */ + public boolean remove(Annotation a) { + return getAnnotations().remove(a); + } + + /** + * Returns the total number of annotations in the tree. + * + * @return the total number of annotations in the tree. + */ + public int numAnnotations() { + return annotations.annotations().size(); + } + + /** + * Clears all Annotations for a given SpanNode. + * + * @param node the SpanNode to clear all Annotations for. + */ + public void clearAnnotations(SpanNode node) { + Iterator<Annotation> annIt = iterator(node); + while (annIt.hasNext()) { + annIt.next(); + annIt.remove(); + } + } + + /** + * Clears all Annotations for a given SpanNode and its child nodes. + * + * @param node the SpanNode to clear all Annotations for. + */ + public void clearAnnotationsRecursive(SpanNode node) { + Iterator<Annotation> annIt = iteratorRecursive(node); + while (annIt.hasNext()) { + annIt.next(); + annIt.remove(); + } + } + + /** + * Returns an Iterator over all annotations in this tree. Note that the iteration order is non-deterministic. + * @return an Iterator over all annotations in this tree. + */ + @SuppressWarnings("unchecked") + public Iterator<Annotation> iterator() { + return annotations.annotations().iterator(); + } + + /** + * Returns an Iterator over all annotations that annotate the given node. + * + * @param node the node to return annotations for. + * @return an Iterator over all annotations that annotate the given node. + */ + @SuppressWarnings("unchecked") + public Iterator<Annotation> iterator(SpanNode node) { + return annotations.iterator(node); + } + + /** + * Returns a recursive Iterator over all annotations that annotate the given node and its subnodes. + * + * @param node the node to recursively return annotations for. + * @return a recursive Iterator over all annotations that annotate the given node and its subnodes. + */ + @SuppressWarnings("unchecked") + public Iterator<Annotation> iteratorRecursive(SpanNode node) { + return annotations.iteratorRecursive(node); + } + + /** + * Returns itself. Needed for this class to be able to be a parent of SpanNodes. + * + * @return this SpanTree instance. + */ + @Override + public SpanTree getSpanTree() { + return this; + } + + /** + * Sets the StringFieldValue that this SpanTree belongs to. This is called by + * {@link StringFieldValue#setSpanTree(SpanTree)} and there is no need for the user to call this + * except in unit tests. + * + * @param stringFieldValue the StringFieldValue that this SpanTree should belong to (might be null to clear the current value) + */ + public void setStringFieldValue(StringFieldValue stringFieldValue) { + this.stringFieldValue = stringFieldValue; + } + + /** + * Returns the StringFieldValue that this SpanTree belongs to. + * + * @return the StringFieldValue that this SpanTree belongs to, if any, otherwise null. + */ + @Override + public StringFieldValue getStringFieldValue() { + return stringFieldValue; + } + + public void createIndex(IndexKey key) { + if (key == IndexKey.SPAN_NODE && annotations instanceof ListAnnotationContainer) { + AnnotationContainer tmpAnnotations = new SpanNode2AnnotationContainer(); + tmpAnnotations.annotateAll(annotations.annotations()); + annotations = tmpAnnotations; + } else if (key == IndexKey.ANNOTATION_TYPE && annotations instanceof ListAnnotationContainer) { + AnnotationContainer tmpAnnotations = new AnnotationType2AnnotationContainer(); + tmpAnnotations.annotateAll(annotations.annotations()); + annotations = tmpAnnotations; + } else { + throw new IllegalArgumentException("Multiple indexes not yet supported. Use clearIndex() or clearIndexes() first."); + } + } + + public void clearIndex(IndexKey key) { + if (key == IndexKey.SPAN_NODE && annotations instanceof SpanNode2AnnotationContainer) { + clearIndex(); + } else if (key == IndexKey.ANNOTATION_TYPE && annotations instanceof AnnotationType2AnnotationContainer) { + clearIndex(); + } + } + + public void clearIndexes() { + if (!(annotations instanceof ListAnnotationContainer)) { + clearIndex(); + } + } + + private void clearIndex() { + AnnotationContainer tmpAnnotations = new ListAnnotationContainer(); + tmpAnnotations.annotateAll(annotations.annotations()); + annotations = tmpAnnotations; + } + + public Collection<IndexKey> getCurrentIndexes() { + if (annotations instanceof AnnotationType2AnnotationContainer) + return ImmutableList.of(IndexKey.ANNOTATION_TYPE); + if (annotations instanceof SpanNode2AnnotationContainer) + return ImmutableList.of(IndexKey.SPAN_NODE); + return ImmutableList.of(); + } + + @Override + public String toString() { + return "SpanTree '" + name + "'"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SpanTree)) return false; + + SpanTree tree = (SpanTree) o; + if (!annotationsEquals(tree)) return false; + if (!name.equals(tree.name)) return false; + if (!root.equals(tree.root)) return false; + + return true; + } + + @SuppressWarnings("unchecked") + private boolean annotationsEquals(SpanTree tree) { + List<Annotation> annotationCollection = new LinkedList<Annotation>(getAnnotations()); + List<Annotation> otherAnnotations = new LinkedList<Annotation>(tree.getAnnotations()); + + return annotationCollection.size() == otherAnnotations.size() && CollectionUtils.isEqualCollection(annotationCollection, otherAnnotations); + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + root.hashCode(); + result = 31 * result + annotations.hashCode(); + return result; + } + + @Override + public int compareTo(SpanTree spanTree) { + int comp = name.compareTo(spanTree.name); + if (comp != 0) { + comp = root.compareTo(spanTree.root); + } + return comp; + } + + public enum IndexKey { + SPAN_NODE, + ANNOTATION_TYPE + } +} diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanTrees.java b/document/src/main/java/com/yahoo/document/annotation/SpanTrees.java new file mode 100644 index 00000000000..449e803a248 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/SpanTrees.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.annotation; + +/** + * This is a container for all {@link SpanTree}s constants used by built-in Vespa features. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@SuppressWarnings({ "UnusedDeclaration" }) +// TODO: Remove. This is the wrong place. +public final class SpanTrees { + + private SpanTrees() { + // unreachable + } + + public static final String LINGUISTICS = "linguistics"; +} diff --git a/document/src/main/java/com/yahoo/document/annotation/package-info.java b/document/src/main/java/com/yahoo/document/annotation/package-info.java new file mode 100644 index 00000000000..235252cd030 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/annotation/package-info.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Provides classes and interfaces for creating trees of spans over string + * values in Vespa documents, and annotating these spans. + */ +@ExportPackage +@PublicApi +package com.yahoo.document.annotation; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/config/package-info.java b/document/src/main/java/com/yahoo/document/config/package-info.java new file mode 100644 index 00000000000..d28ae619a10 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/config/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.document.config; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java new file mode 100644 index 00000000000..66cc472de69 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java @@ -0,0 +1,543 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.collections.CollectionComparator; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.FieldPath; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; + +import java.util.*; + +/** + * FieldValue which encapsulates a Array value + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public final class Array<T extends FieldValue> extends CollectionFieldValue<T> implements List<T> { + + private List<T> values; + + public Array(DataType type) { + this(type, 1); + } + + public Array(DataType type, int initialCapacity) { + super((ArrayDataType) type); + this.values = new ArrayList<>(initialCapacity); + } + + public Array(DataType type, List<T> values) { + this(type); + for (T v : values) { + if (!((ArrayDataType)type).getNestedType().isValueCompatible(v)) { + throw new IllegalArgumentException("FieldValue " + v + + " is not compatible with " + type + "."); + } + } + this.values.addAll(values); + } + + @Override + public ArrayDataType getDataType() { + return (ArrayDataType) super.getDataType(); + } + + @Override + public Iterator<T> fieldValueIterator() { + return values.iterator(); + } + + @Override + public Array<T> clone() { + Array<T> array = (Array<T>) super.clone(); + array.values = new ArrayList<>(values.size()); + for (T fval : values) { + array.values.add((T) fval.clone()); + } + return array; + } + + @Override + public void clear() { + values.clear(); + } + + @Override + public void assign(Object o) { + if (!checkAssign(o)) { + return; + } + + if (o instanceof Array) { + if (o == this) return; + Array a = (Array) o; + values.clear(); + addAll(a.values); + } else if (o instanceof List) { + values = new ListWrapper<T>((List) o); + } else { + throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + @Override + public Object getWrappedValue() { + if (values instanceof ListWrapper) { + return ((ListWrapper) values).myvalues; + } + List tmpWrappedList = new ArrayList(); + for (T value : values) { + tmpWrappedList.add(value.getWrappedValue()); + } + return tmpWrappedList; + } + + public List<T> getValues() { + return values; + } + + public FieldValue getFieldValue(int index) { + return values.get(index); + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printArrayXml(this, xml); + } + + @Override + public String toString() { + return values.toString(); + } + + @Override + public int hashCode() { + int hashCode = super.hashCode(); + for (FieldValue val : values) { + hashCode ^= val.hashCode(); + } + return hashCode; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Array)) return false; + if (!super.equals(o)) return false; + Array a = (Array) o; + // Compare independent of container used. + Iterator it1 = values.iterator(); + Iterator it2 = a.values.iterator(); + while (it1.hasNext() && it2.hasNext()) { + if (!it1.next().equals(it2.next())) return false; + } + return !(it1.hasNext() || it2.hasNext()); + } + + // List implementation + + public void add(int index, T o) { + verifyElementCompatibility(o); + values.add(index, o); + } + + public boolean remove(Object o) { + return values.remove(o); + } + + public boolean add(T o) { + verifyElementCompatibility(o); + return values.add(o); + } + + @Override + public boolean contains(Object o) { + return values.contains(o); + } + + @Override + public boolean isEmpty() { + return super.isEmpty(values); + } + + @Override + public Iterator<T> iterator() { + return values.iterator(); + } + + @Override + public boolean removeValue(FieldValue o) { + return super.removeValue(o, values); + } + + @Override + public int size() { + return values.size(); + } + + public boolean addAll(Collection<? extends T> c) { + for (T t : c) { + verifyElementCompatibility(t); + } + return values.addAll(c); + } + + public boolean containsAll(Collection<?> c) { + return values.containsAll(c); + } + + public Object[] toArray() { + return values.toArray(); + } + + @SuppressWarnings({"unchecked"}) + public <T> T[] toArray(T[] a) { + return values.toArray(a); + } + + public boolean addAll(int index, Collection<? extends T> c) { + for (T t : c) { + verifyElementCompatibility(t); + } + return values.addAll(index, c); + } + + @SuppressWarnings("deprecation") + public boolean retainAll(Collection<?> c) { + return values.retainAll(c); + } + + @SuppressWarnings("deprecation") + public boolean removeAll(Collection<?> c) { + return values.removeAll(c); + } + + public T get(int index) { + return values.get(index); + } + + @SuppressWarnings("deprecation") + public int indexOf(Object o) { + return values.indexOf(o); + } + + @SuppressWarnings("deprecation") + public int lastIndexOf(Object o) { + return values.lastIndexOf(o); + } + + public ListIterator<T> listIterator() { + return values.listIterator(); + } + + public ListIterator<T> listIterator(final int index) { + return values.listIterator(index); + } + + public T remove(int index) { + return values.remove(index); + } + + @SuppressWarnings("deprecation") + public T set(int index, T o) { + verifyElementCompatibility(o); + T fval = values.set(index, o); + return fval; + } + + public List<T> subList(int fromIndex, int toIndex) { + return values.subList(fromIndex, toIndex); + } + + FieldPathIteratorHandler.ModificationStatus iterateSubset(int startPos, int endPos, FieldPath fieldPath, String variable, int nextPos, FieldPathIteratorHandler handler) { + FieldPathIteratorHandler.ModificationStatus retVal = FieldPathIteratorHandler.ModificationStatus.NOT_MODIFIED; + + LinkedList<Integer> indicesToRemove = new LinkedList<Integer>(); + + for (int i = startPos; i <= endPos && i < values.size(); i++) { + if (variable != null) { + handler.getVariables().put(variable, new FieldPathIteratorHandler.IndexValue(i)); + } + + FieldValue fv = values.get(i); + FieldPathIteratorHandler.ModificationStatus status = fv.iterateNested(fieldPath, nextPos, handler); + + if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) { + indicesToRemove.addFirst(i); + retVal = FieldPathIteratorHandler.ModificationStatus.MODIFIED; + } else if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) { + retVal = status; + } + } + + if (variable != null) { + handler.getVariables().remove(variable); + } + + for (Integer idx : indicesToRemove) { + values.remove(idx.intValue()); + } + return retVal; + } + + @Override + FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler) { + if (pos < fieldPath.size()) { + switch (fieldPath.get(pos).getType()) { + case ARRAY_INDEX: + return iterateSubset(fieldPath.get(pos).getLookupIndex(), fieldPath.get(pos).getLookupIndex(), fieldPath, null, pos + 1, handler); + case VARIABLE: { + FieldPathIteratorHandler.IndexValue val = handler.getVariables().get(fieldPath.get(pos).getVariableName()); + if (val != null) { + int idx = val.getIndex(); + + if (idx == -1) { + throw new IllegalArgumentException("Mismatch between variables - trying to iterate through map and array with the same variable."); + } + + if (idx < values.size()) { + return iterateSubset(idx, idx, fieldPath, null, pos + 1, handler); + } + } else { + return iterateSubset(0, values.size() - 1, fieldPath, fieldPath.get(pos).getVariableName(), pos + 1, handler); + } + break; + } + default: + } + return iterateSubset(0, values.size() - 1, fieldPath, null, pos, handler); + } else { + FieldPathIteratorHandler.ModificationStatus status = handler.modify(this); + + if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) { + return status; + } + + if (handler.onComplex(this)) { + if (iterateSubset(0, values.size() - 1, fieldPath, null, pos, handler) != FieldPathIteratorHandler.ModificationStatus.NOT_MODIFIED) { + status = FieldPathIteratorHandler.ModificationStatus.MODIFIED; + } + } + + return status; + } + } + + /** + * This wrapper class is used to wrap a list that isn't a list of field + * values. This is done, as to not alter behaviour from previous state, + * where people could add whatever list to a document, and then keep adding + * stuff to the list afterwards. + * + * <p> + * TODO: Remove this class and only allow instance of Array to be added. + */ + class ListWrapper<E> implements List<E>, RandomAccess { + private final List myvalues; + + private Object unwrap(Object o) { + return (o instanceof FieldValue ? ((FieldValue) o).getWrappedValue() : o); + } + + public ListWrapper(List wrapped) { + myvalues = wrapped; + } + + public int size() { + return myvalues.size(); + } + + public boolean isEmpty() { + return myvalues.isEmpty(); + } + + public boolean contains(Object o) { + return myvalues.contains(unwrap(o)); + } + + public Iterator<E> iterator() { + return listIterator(); + } + + public Object[] toArray() { + return toArray(new Object[myvalues.size()]); + } + + // It's supposed to blow up if given invalid types + @SuppressWarnings({ "hiding", "unchecked" }) + public <T> T[] toArray(T[] a) { + final Class<?> componentType = a.getClass().getComponentType(); + T[] out = (T[]) java.lang.reflect.Array.newInstance(componentType, myvalues.size()); + + Arrays.setAll(out, (i) -> (T) createFieldValue(myvalues.get(i))); + return out; + } + + public boolean add(E o) { + return myvalues.add(unwrap(o)); + } + + public boolean remove(Object o) { + return myvalues.remove(unwrap(o)); + } + + public boolean containsAll(Collection<?> c) { + for (Object o : c) { + if (!myvalues.contains(unwrap(o))) return false; + } + return true; + } + + public boolean addAll(Collection<? extends E> c) { + boolean result = false; + for (Object o : c) { + result |= myvalues.add(unwrap(o)); + } + return result; + } + + public boolean addAll(int index, Collection<? extends E> c) { + for (Object o : c) { + myvalues.add(index++, unwrap(o)); + } + return true; + } + + public boolean removeAll(Collection<?> c) { + boolean result = false; + for (Object o : c) { + result |= myvalues.remove(unwrap(o)); + } + return result; + } + + public boolean retainAll(Collection<?> c) { + throw new UnsupportedOperationException("retainAll() not implemented for this type"); + } + + public void clear() { + myvalues.clear(); + } + + public E get(int index) { + Object o = myvalues.get(index); + return (E) (o == null ? null : createFieldValue(o)); + } + + public E set(int index, E element) { + Object o = myvalues.set(index, unwrap(element)); + return (E) (o == null ? null : createFieldValue(o)); + } + + public void add(int index, E element) { + myvalues.add(index, unwrap(element)); + } + + public E remove(int index) { + Object o = myvalues.remove(index); + return (E) (o == null ? null : createFieldValue(o)); + } + + public int indexOf(Object o) { + return myvalues.indexOf(unwrap(o)); + } + + public int lastIndexOf(Object o) { + return myvalues.lastIndexOf(unwrap(o)); + } + + public ListIterator<E> listIterator() { + return listIterator(0); + } + + public ListIterator<E> listIterator(final int index) { + return new ListIterator<E>() { + ListIterator it = myvalues.listIterator(index); + + public boolean hasNext() { + return it.hasNext(); + } + + public E next() { + return (E) createFieldValue(it.next()); + } + + public boolean hasPrevious() { + return it.hasPrevious(); + } + + public E previous() { + return (E) createFieldValue(it.previous()); + } + + public int nextIndex() { + return it.nextIndex(); + } + + public int previousIndex() { + return it.previousIndex(); + } + + public void remove() { + it.remove(); + } + + public void set(E o) { + it.set(unwrap(o)); + } + + public void add(E o) { + it.add(unwrap(o)); + } + }; + } + + @SuppressWarnings("deprecation") + public List<E> subList(int fromIndex, int toIndex) { + return new ListWrapper<E>(myvalues.subList(fromIndex, toIndex)); + } + + public String toString() { + return myvalues.toString(); + } + + @Override + @SuppressWarnings("deprecation, unchecked") + public boolean equals(Object o) { + return this == o || o instanceof ListWrapper && myvalues.equals(((ListWrapper) o).myvalues); + } + + @Override + public int hashCode() { + return myvalues.hashCode(); + } + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + + //types are equal, this must be of this type + Array otherValue = (Array) fieldValue; + return CollectionComparator.compare(values, otherValue.values); + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java new file mode 100644 index 00000000000..c6606dd9886 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/ByteFieldValue.java @@ -0,0 +1,154 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +/** + * FieldValue which encapsulates a byte. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ByteFieldValue extends NumericFieldValue { + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new ByteFieldValue(); + } + } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 10, ByteFieldValue.class); + private byte value; + + public ByteFieldValue() { + this((byte) 0); + } + + public ByteFieldValue(byte value) { + this.value = value; + } + + public ByteFieldValue(Byte value) { + this.value = value; + } + + public ByteFieldValue(Integer value) { + this.value = (byte) value.intValue(); + } + + public ByteFieldValue(String s) { value = Byte.parseByte(s); } + + @Override + public ByteFieldValue clone() { + ByteFieldValue val = (ByteFieldValue) super.clone(); + val.value = value; + return val; + + } + + @Override + public Number getNumber() { + return value; + } + + @Override + public void clear() { + value = (byte) 0; + } + + @Override + public void assign(Object o) { + if (!checkAssign(o)) { + return; + } + if (o instanceof Number) { + value = ((Number) o).byteValue(); + } else if (o instanceof NumericFieldValue) { + value = ((NumericFieldValue) o).getNumber().byteValue(); + } else if (o instanceof String || o instanceof StringFieldValue) { + value = Byte.parseByte(o.toString()); + } else { + throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + public byte getByte() { + return value; + } + + @Override + public Object getWrappedValue() { + return value; + } + + @Override + public DataType getDataType() { + return DataType.BYTE; + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printByteXml(this, xml); + } + + @Override + public String toString() { + return "" + value; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (int) value; + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ByteFieldValue)) return false; + if (!super.equals(o)) return false; + + ByteFieldValue that = (ByteFieldValue) o; + if (value != that.value) return false; + return true; + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + /* (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + + //types are equal, this must be of this type + ByteFieldValue otherValue = (ByteFieldValue) fieldValue; + if (value < otherValue.value) { + return -1; + } else if (value > otherValue.value) { + return 1; + } else { + return 0; + } + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java new file mode 100644 index 00000000000..aeeb4abf632 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/CollectionFieldValue.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.CollectionDataType; + +import java.util.Collection; +import java.util.Iterator; + +/** + * Date: Apr 16, 2008 + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public abstract class CollectionFieldValue<T extends FieldValue> extends CompositeFieldValue { + + CollectionFieldValue(CollectionDataType type) { + super(type); + } + + @Override + public CollectionDataType getDataType() { + return (CollectionDataType) super.getDataType(); + } + + /** + * Utility function to wrap primitives. + * + * @see Array.ListWrapper + */ + protected FieldValue createFieldValue(Object o) { + if (o instanceof FieldValue) { + if (!getDataType().getNestedType().isValueCompatible((FieldValue) o)) { + throw new IllegalArgumentException( + "Incompatible data types. Got " + + ((FieldValue)o).getDataType() + ", expected " + + getDataType().getNestedType()); + } + return (FieldValue) o; + } else { + FieldValue fval = getDataType().getNestedType().createFieldValue(); + fval.assign(o); + return fval; + } + } + + public void verifyElementCompatibility(T o) { + if (!getDataType().getNestedType().isValueCompatible(o)) { + throw new IllegalArgumentException( + "Incompatible data types. Got " + + o.getDataType() + ", expected " + + getDataType().getNestedType()); + } + } + + public abstract Iterator<T> fieldValueIterator(); + + // Collection implementation + + public abstract boolean add(T value); + + public abstract boolean contains(Object o); + + public abstract boolean isEmpty(); + + protected boolean isEmpty(Collection collection) { + return collection.isEmpty(); + } + + public abstract Iterator<T> iterator(); + + public abstract boolean removeValue(FieldValue o); + + protected boolean removeValue(FieldValue o, Collection collection) { + int removedCount = 0; + while (collection.remove(o)) { + ++removedCount; + } + return (removedCount > 0); + } + + public abstract int size(); +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java new file mode 100644 index 00000000000..a1f0f81eea7 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/CompositeFieldValue.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; + +public abstract class CompositeFieldValue extends FieldValue { + private DataType dataType; + + public CompositeFieldValue(DataType dataType) { + this.dataType = dataType; + } + + @Override + public DataType getDataType() { + return dataType; + } + + public void setDataType(DataType dataType) { + this.dataType = dataType; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof CompositeFieldValue)) return false; + if (!super.equals(o)) return false; + + CompositeFieldValue that = (CompositeFieldValue) o; + if (dataType != null ? !dataType.equals(that.dataType) : that.dataType != null) return false; + return true; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (dataType != null ? dataType.hashCode() : 0); + return result; + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java new file mode 100644 index 00000000000..2d9d900a092 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/DoubleFieldValue.java @@ -0,0 +1,145 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +/** + * FieldValue which encapsulates a double. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public final class DoubleFieldValue extends NumericFieldValue { + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new DoubleFieldValue(); + } + } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 14, DoubleFieldValue.class); + private double value; + + public DoubleFieldValue() { + this(0.0); + } + + public DoubleFieldValue(double value) { + this.value = value; + } + + public DoubleFieldValue(Double value) { + this.value = value; + } + + public DoubleFieldValue(String s) { value = Double.parseDouble(s); } + + @Override + public DoubleFieldValue clone() { + DoubleFieldValue val = (DoubleFieldValue) super.clone(); + val.value = value; + return val; + } + + @Override + public void clear() { + value = 0.0; + } + + @Override + public Number getNumber() { + return value; + } + + @Override + public void assign(Object obj) { + if (!checkAssign(obj)) { + return; + } + if (obj instanceof Number) { + value = ((Number) obj).doubleValue(); + } else if (obj instanceof NumericFieldValue) { + value = (((NumericFieldValue) obj).getNumber().doubleValue()); + } else if (obj instanceof String || obj instanceof StringFieldValue) { + value = Double.parseDouble(obj.toString()); + } else { + throw new IllegalArgumentException("Class " + obj.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + public double getDouble() { + return value; + } + + @Override + public Object getWrappedValue() { + return value; + } + + @Override + public DataType getDataType() { + return DataType.DOUBLE; + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printDoubleXml(this, xml); + } + + @Override + public String toString() { + return "" + value; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + long temp; + temp = value != +0.0d ? Double.doubleToLongBits(value) : 0L; + result = 31 * result + (int) (temp ^ (temp >>> 32)); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DoubleFieldValue)) return false; + if (!super.equals(o)) return false; + + DoubleFieldValue that = (DoubleFieldValue) o; + if (Double.compare(that.value, value) != 0) return false; + return true; + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + /* (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + + //types are equal, this must be of this type + DoubleFieldValue otherValue = (DoubleFieldValue) fieldValue; + return Double.compare(value, otherValue.value); + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java b/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java new file mode 100644 index 00000000000..c8d007037f3 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/FieldPathIteratorHandler.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +/** + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public abstract class FieldPathIteratorHandler { + + public static class IndexValue { + + private int index; + private FieldValue key; + + public int getIndex() { + return index; + } + + public FieldValue getKey() { + return key; + } + + public IndexValue() { + index = -1; + key = null; + } + + public IndexValue(int index) { + this.index = index; + key = null; + } + + public IndexValue(FieldValue key) { + index = -1; + this.key = key; + } + + public String toString() { + if (key != null) { + return key.toString(); + } else { + return "" + index; + } + } + + @Override + public boolean equals(Object o) { + IndexValue other = (IndexValue)o; + + if (key != null) { + if (other.key != null && key.equals(other.key)) { + return true; + } + return false; + } + + return index == other.index; + } + }; + + public static class VariableMap extends TreeMap<String, IndexValue> { + + @Override + public Object clone() { + Map<String, IndexValue> map = new VariableMap(); + map.putAll(this); + return map; + } + } + + private VariableMap variables = new VariableMap(); + + public void onPrimitive(FieldValue fv) { + + } + + public boolean onComplex(FieldValue fv) { + return true; + } + + public ModificationStatus doModify(FieldValue fv) { + return ModificationStatus.NOT_MODIFIED; + } + + public enum ModificationStatus { + MODIFIED, REMOVED, NOT_MODIFIED + } + + public ModificationStatus modify(FieldValue fv) { + return doModify(fv); + } + + public boolean createMissingPath() { + return false; + } + + public VariableMap getVariables() { + return variables; + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java new file mode 100644 index 00000000000..df10bca83a2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java @@ -0,0 +1,187 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.FieldPath; +import com.yahoo.document.serialization.*; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.vespa.objects.BufferSerializer; +import com.yahoo.vespa.objects.Deserializer; +import com.yahoo.vespa.objects.Identifiable; +import com.yahoo.vespa.objects.Ids; +import com.yahoo.vespa.objects.Serializer; +import com.yahoo.document.config.DocumentmanagerConfig.Datatype.Structtype.Compresstype; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public abstract class FieldValue extends Identifiable implements Comparable<FieldValue> { + + public static final int classId = registerClass(Ids.document + 9, FieldValue.class); + + public abstract DataType getDataType(); + + public static FieldValue create(FieldReader reader, DataType type) { + FieldValue value = type.createFieldValue(); + value.deserialize(reader); + return value; + } + + /** + * Get XML representation of a single field and all its children, if any. + * @return XML representation of field in a <value> element + */ + public String toXml() { + XmlStream xml = new XmlStream(); + xml.setIndent(" "); + xml.beginTag("value"); + printXml(xml); + xml.endTag(); + return xml.toString(); + } + + /** + * Read data from the given buffer to create this field value. As some field values have their type self + * contained, we need the type manager object to be able to retrieve it. + */ + final public void deserialize(FieldReader reader) { + deserialize(null, reader); + } + + final public void serialize(GrowableByteBuffer buf) { + serialize(DocumentSerializerFactory.create42(buf)); + } + + public abstract void printXml(XmlStream xml); + + public abstract void clear(); + + @Override + public FieldValue clone() { + return (FieldValue) super.clone(); + } + + boolean checkAssign(Object o) { + if (o == null) { + clear(); + return false; + } + + return true; + } + + /** + * Assign this non-fieldvalue value to this field value. This is used to be able + * to assign ints to Integer field values and List to Array field values and such. + * <p> + * Override to accept the specific types that should be legal. + * + * @throws IllegalArgumentException If the object given is of wrong type for this field value. + */ + public abstract void assign(Object o); + + /** + * Used to retrieve wrapped type for simple types, such that you can use get methods to retrieve ints and floats + * directly instead of Int/Float field values. Complex types that can't be specified by simple java types just + * return themself. + */ + public Object getWrappedValue() { + return this; + } + + class RecursiveIteratorHandler extends FieldPathIteratorHandler { + FieldValue retVal = null; + boolean multiValue = false; + + @Override + public boolean onComplex(FieldValue fv) { + onPrimitive(fv); + return false; + } + + @Override + public void onPrimitive(FieldValue fv) { + if (retVal != null) { + if (multiValue) { + ((Array) retVal).add(fv); + } else { + Array afv = new Array(new ArrayDataType(retVal.getDataType())); + afv.add(retVal); + afv.add(fv); + retVal = afv; + multiValue = true; + } + } else { + retVal = fv; + } + } + } + + /** + * Using the given field path, digs through the document and returns the matching field value. + * If the field path resolves to multiple values, returns an ArrayFieldValue containing the + * values. + */ + public FieldValue getRecursiveValue(String path) { + return getRecursiveValue(getDataType().buildFieldPath(path)); + } + + public FieldValue getRecursiveValue(FieldPath path) { + RecursiveIteratorHandler handler = new RecursiveIteratorHandler(); + iterateNested(path, 0, handler); + return handler.retVal; + } + + @Override + public void onSerialize(Serializer target) { + if (target instanceof FieldWriter) { + serialize(null, (FieldWriter) target); + } else if (target instanceof BufferSerializer) { + serialize(null, DocumentSerializerFactory.create42(((BufferSerializer) target).getBuf())); + } else { + DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer()); + serialize(null, fw); + target.put(null, fw.getBuf().getByteBuffer()); + } + } + + @Override + public void onDeserialize(Deserializer data) { + if (data instanceof FieldReader) { + deserialize(null, (FieldReader) data); + } else { + throw new IllegalArgumentException("I am not able to deserialize from " + data.getClass().getName()); + } + } + + /** + * Iterates through the document using the given fieldpath, calling callbacks in the given iterator + * handler. + */ + FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler) { + if (pos >= fieldPath.size()) { + handler.onPrimitive(this); + return handler.modify(this); + } else { + throw new IllegalArgumentException("Primitive types can't be iterated through"); + } + } + + /** + * Write out field value to the specified writer + */ + abstract public void serialize(Field field, FieldWriter writer); + + /** + * Read a field value from the specified reader + */ + abstract public void deserialize(Field field, FieldReader reader); + + @Override + public int compareTo(FieldValue fieldValue) { + return getDataType().compareTo(fieldValue.getDataType()); + } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java new file mode 100644 index 00000000000..a8c83f426ed --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/FloatFieldValue.java @@ -0,0 +1,144 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +/** + * FieldValue which encapsulates a float. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public final class FloatFieldValue extends NumericFieldValue { + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new FloatFieldValue(); + } + } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 13, FloatFieldValue.class); + private float value; + + public FloatFieldValue() { + this((float) 0); + } + + public FloatFieldValue(float value) { + this.value = value; + } + + public FloatFieldValue(Float value) { + this.value = value; + } + + public FloatFieldValue(String s) { value = Float.parseFloat(s); } + + @Override + public FloatFieldValue clone() { + FloatFieldValue val = (FloatFieldValue) super.clone(); + val.value = value; + return val; + } + + @Override + public Number getNumber() { + return value; + } + + @Override + public void clear() { + value = 0.0f; + } + + @Override + public void assign(Object obj) { + if (!checkAssign(obj)) { + return; + } + if (obj instanceof Number) { + value = ((Number) obj).floatValue(); + } else if (obj instanceof NumericFieldValue) { + value = (((NumericFieldValue) obj).getNumber().floatValue()); + } else if (obj instanceof String || obj instanceof StringFieldValue) { + value = Float.parseFloat(obj.toString()); + } else { + throw new IllegalArgumentException("Class " + obj.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + public float getFloat() { + return value; + } + + @Override + public Object getWrappedValue() { + return value; + } + + @Override + public DataType getDataType() { + return DataType.FLOAT; + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printFloatXml(this, xml); + } + + @Override + public String toString() { + return "" + value; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (value != +0.0f ? Float.floatToIntBits(value) : 0); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof FloatFieldValue)) return false; + if (!super.equals(o)) return false; + + FloatFieldValue that = (FloatFieldValue) o; + if (Float.compare(that.value, value) != 0) return false; + return true; + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + /* (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + + //types are equal, this must be of this type + FloatFieldValue otherValue = (FloatFieldValue) fieldValue; + return Float.compare(value, otherValue.value); + } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java new file mode 100644 index 00000000000..4f54c4c6cb1 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/IntegerFieldValue.java @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +/** + * FieldValue which encapsulates an int. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public final class IntegerFieldValue extends NumericFieldValue { + + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new IntegerFieldValue(); + } + } + + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 11, IntegerFieldValue.class); + private int value; + + public IntegerFieldValue() { + this(0); + } + + public IntegerFieldValue(int value) { + this.value = value; + } + + public IntegerFieldValue(Integer value) { + this.value = value; + } + + public IntegerFieldValue(String s) { + value = Integer.parseInt(s); + } + + @Override + public IntegerFieldValue clone() { + IntegerFieldValue val = (IntegerFieldValue) super.clone(); + val.value = value; + return val; + } + + @Override + public Number getNumber() { + return value; + } + + @Override + public void clear() { + value = 0; + } + + @Override + public void assign(Object obj) { + if (!checkAssign(obj)) { + return; + } + if (obj instanceof Number) { + value = ((Number) obj).intValue(); + } else if (obj instanceof NumericFieldValue) { + value = (((NumericFieldValue) obj).getNumber().intValue()); + } else if (obj instanceof String || obj instanceof StringFieldValue) { + value = Integer.parseInt(obj.toString()); + } else { + throw new IllegalArgumentException("Class " + obj.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + public int getInteger() { + return value; + } + + @Override + public Object getWrappedValue() { + return value; + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printIntegerXml(this, xml); + } + + @Override + public String toString() { + return "" + value; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + value; + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof IntegerFieldValue)) return false; + if (!super.equals(o)) return false; + + IntegerFieldValue that = (IntegerFieldValue) o; + return (value == that.value); + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + /* (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public DataType getDataType() { + return DataType.INT; + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + + //types are equal, this must be of this type + IntegerFieldValue otherValue = (IntegerFieldValue) fieldValue; + if (value < otherValue.value) { + return -1; + } else if (value > otherValue.value) { + return 1; + } else { + return 0; + } + } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java new file mode 100644 index 00000000000..3705d89e146 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +/** + * FieldValue which encapsulates a long. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public final class LongFieldValue extends NumericFieldValue { + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new LongFieldValue(); + } + } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 12, LongFieldValue.class); + private long value; + + public LongFieldValue() { + this(0l); + } + + public LongFieldValue(long value) { + this.value = value; + } + + public LongFieldValue(Long value) { + this.value = value; + } + + public LongFieldValue(String s) { + value = Long.parseLong(s); + } + + @Override + public LongFieldValue clone() { + LongFieldValue val = (LongFieldValue) super.clone(); + val.value = value; + return val; + } + + @Override + public void clear() { + value = 0l; + } + + @Override + public Number getNumber() { + return value; + } + + @Override + public void assign(Object obj) { + if (!checkAssign(obj)) { + return; + } + if (obj instanceof Number) { + value = ((Number) obj).longValue(); + } else if (obj instanceof NumericFieldValue) { + value = (((NumericFieldValue) obj).getNumber().longValue()); + } else if (obj instanceof String || obj instanceof StringFieldValue) { + value = Long.parseLong(obj.toString()); + } else { + throw new IllegalArgumentException("Class " + obj.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + public long getLong() { + return value; + } + + @Override + public Object getWrappedValue() { + return value; + } + + @Override + public DataType getDataType() { + return DataType.LONG; + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printLongXml(this, xml); + } + + @Override + public String toString() { + return "" + value; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (int) (value ^ (value >>> 32)); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof LongFieldValue)) return false; + if (!super.equals(o)) return false; + + LongFieldValue that = (LongFieldValue) o; + if (value != that.value) return false; + return true; + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + /* (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + + //types are equal, this must be of this type + LongFieldValue otherValue = (LongFieldValue) fieldValue; + if (value < otherValue.value) { + return -1; + } else if (value > otherValue.value) { + return 1; + } else { + return 0; + } + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java new file mode 100644 index 00000000000..261d40161d0 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java @@ -0,0 +1,396 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.collections.CollectionComparator; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.FieldPath; +import com.yahoo.document.MapDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import java.util.*; + +/** + * Vespa map. Backed by and and parametrized by FieldValue + * + * @author vegardh + */ +public class MapFieldValue<K extends FieldValue, V extends FieldValue> extends CompositeFieldValue implements java.util.Map<K,V> { + + private java.util.Map<K,V> values; + + public MapFieldValue(MapDataType type) { + this(type, 1); + } + + public MapFieldValue(MapDataType type, int initialCapacity) { + super(type); + values = new HashMap<K, V>(initialCapacity); + } + + @Override + public MapDataType getDataType() { + return (MapDataType)super.getDataType(); + } + + @Override + public void assign(Object o) { + if (!checkAssign(o)) { + return; + } + + if (o instanceof MapFieldValue) { + if (o == this) return; + MapFieldValue a = (MapFieldValue) o; + values.clear(); + putAll(a); + } else if (o instanceof Map) { + values = new MapWrapper((Map)o); + } + else { + throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + @Override + public MapFieldValue clone() { + MapFieldValue copy = (MapFieldValue) super.clone(); + copy.values = new HashMap<K, V>(values.size()); + for (Map.Entry<K, V> entry : values.entrySet()) { + copy.values.put(entry.getKey().clone(), entry.getValue().clone()); + } + return copy; + } + + /** + * Checks if another object is equal to this set. + * + * @param o the object to check for equality with + * @return true if o is an instance of WeightedSet and the two encapsulated Maps are equal, false otherwise + */ + public boolean equals(Object o) { + if (!(o instanceof MapFieldValue)) return false; + MapFieldValue otherSet = (MapFieldValue) o; + Map<K, V> map1 = values; + Map<K, V> map2 = otherSet.values; + return (super.equals(o) && map1.equals(map2)); + } + + @Override + public void clear() { + values.clear(); + } + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printMapXml(this, xml); + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + @Override + public Object getWrappedValue() { + if (values instanceof MapFieldValue.MapWrapper) { + return ((MapFieldValue.MapWrapper) values).map; + } + Map tmpMap = new HashMap(); + for (Entry<K, V> kvEntry : values.entrySet()) { + tmpMap.put(kvEntry.getKey().getWrappedValue(), kvEntry.getValue().getWrappedValue()); + } + return tmpMap; + } + + ///// java.util.Map methods + + public boolean containsKey(Object key) { + return values.containsKey(key); + } + + public boolean containsValue(Object value) { + return values.containsValue(value); + } + + public Set<java.util.Map.Entry<K, V>> entrySet() { + return values.entrySet(); + } + + public V get(Object key) { + return values.get(key); + } + + public Set<K> keySet() { + return values.keySet(); + } + + private void validateCompatibleTypes(DataType d, FieldValue v) { + if (!d.isValueCompatible(v)) { + throw new IllegalArgumentException( + "Incompatible data types. Got " + v.getDataType() + + ", expected " + d); + } + } + + public V put(K key, V value) { + validateCompatibleTypes(getDataType().getKeyType(), key); + validateCompatibleTypes(getDataType().getValueType(), value); + return values.put(key, value); + } + + public void putAll(java.util.Map<? extends K, ? extends V> m) { + for (K key : m.keySet()) { + validateCompatibleTypes(getDataType().getKeyType(), key); + } + for (V value : m.values()) { + validateCompatibleTypes(getDataType().getValueType(), value); + } + values.putAll(m); + } + + public V remove(Object key) { + return values.remove(key); + } + + public Collection<V> values() { + return values.values(); + } + + public boolean contains(Object o) { + return values.containsKey(o); + } + + public boolean isEmpty() { + return values.isEmpty(); + } + + public int size() { + return values.size(); + } + + boolean checkAndRemove(FieldValue key, FieldPathIteratorHandler.ModificationStatus status, boolean wasModified, List<FieldValue> keysToRemove) { + if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) { + keysToRemove.add(key); + return true; + } else if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) { + return true; + } + + return wasModified; + } + + FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler, FieldValue complexFieldValue) { + List<FieldValue> keysToRemove = new ArrayList<FieldValue>(); + boolean wasModified = false; + + if (pos < fieldPath.size()) { + switch (fieldPath.get(pos).getType()) { + case MAP_KEY: + { + FieldValue val = values.get(fieldPath.get(pos).getLookupKey()); + if (val != null) { + wasModified = checkAndRemove(fieldPath.get(pos).getLookupKey(), val.iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove); + } else if (handler.createMissingPath()) { + val = getDataType().getValueType().createFieldValue(); + FieldPathIteratorHandler.ModificationStatus status = val.iterateNested(fieldPath, pos + 1, handler); + if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) { + put((K)fieldPath.get(pos).getLookupKey(), (V)val); + return status; + } + } + break; + } + case MAP_ALL_KEYS: + for (FieldValue f : values.keySet()) { + wasModified = checkAndRemove(f, f.iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove); + } + break; + case MAP_ALL_VALUES: + for (Map.Entry<K, V> entry : values.entrySet()) { + wasModified = checkAndRemove(entry.getKey(), entry.getValue().iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove); + } + break; + case VARIABLE: + { + FieldPathIteratorHandler.IndexValue idx = handler.getVariables().get(fieldPath.get(pos).getVariableName()); + if (idx != null) { + FieldValue val = values.get(idx.getKey()); + if (val != null) { + wasModified = checkAndRemove(idx.getKey(), val.iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove); + } + } else { + for (Map.Entry<K, V> entry : values.entrySet()) { + handler.getVariables().put(fieldPath.get(pos).getVariableName(), new FieldPathIteratorHandler.IndexValue(entry.getKey())); + wasModified = checkAndRemove(entry.getKey(), entry.getValue().iterateNested(fieldPath, pos + 1, handler), wasModified, keysToRemove); + } + handler.getVariables().remove(fieldPath.get(pos).getVariableName()); + } + break; + } + default: + for (Map.Entry<K, V> entry : values.entrySet()) { + wasModified = checkAndRemove(entry.getKey(), entry.getKey().iterateNested(fieldPath, pos, handler), wasModified, keysToRemove); + } + break; + } + } else { + FieldPathIteratorHandler.ModificationStatus status = handler.modify(complexFieldValue); + if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) { + return status; + } else if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) { + wasModified = true; + } + + if (handler.onComplex(complexFieldValue)) { + for (Map.Entry<K, V> entry : values.entrySet()) { + wasModified = checkAndRemove(entry.getKey(), entry.getKey().iterateNested(fieldPath, pos, handler), wasModified, keysToRemove); + } + } + } + + for (FieldValue f : keysToRemove) { + values.remove(f); + } + + return wasModified ? FieldPathIteratorHandler.ModificationStatus.MODIFIED : FieldPathIteratorHandler.ModificationStatus.NOT_MODIFIED; + } + + @Override + FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler) { + return iterateNested(fieldPath, pos, handler, this); + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + //types are equal, this must be of this type + MapFieldValue otherValue = (MapFieldValue) fieldValue; + comp = CollectionComparator.compare(values.keySet(), otherValue.values.keySet()); + + if (comp != 0) { + return comp; + } + + return CollectionComparator.compare(values.values(), otherValue.values.values()); + } + + /** + * Map of field values backed by a normal map of Java objects + * @author vegardh + * + */ + class MapWrapper implements Map<K,V> { + + private Map<Object,Object> map; // Not field values, basic objects + private DataType keyTypeVespa = getDataType().getKeyType(); + private DataType valTypeVespa = getDataType().getValueType(); + public MapWrapper(Map map) { + this.map=map; + } + + private Object unwrap(Object o) { + return (o instanceof FieldValue ? ((FieldValue) o).getWrappedValue() : o); + } + + private K wrapKey(Object o) { + if (o==null) return null; + return (K)keyTypeVespa.createFieldValue(o); + } + + private V wrapValue(Object o) { + if (o==null) return null; + return (V)valTypeVespa.createFieldValue(o); + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public boolean containsKey(Object key) { + return map.containsKey(unwrap(key)); + } + + @Override + public boolean containsValue(Object value) { + return map.containsValue(unwrap(value)); + } + + @Override + public Set<java.util.Map.Entry<K, V>> entrySet() { + Map<K, V> ret = new HashMap<K, V>(); + for (Map.Entry e : map.entrySet()) { + ret.put(wrapKey(e.getKey()), wrapValue(e.getValue())); + } + return ret.entrySet(); + } + + @Override + public V get(Object key) { + Object o = map.get(unwrap(key)); + return o == null ? null : wrapValue(o); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public Set<K> keySet() { + Set<K> ret = new HashSet<K>(); + for (Map.Entry e : map.entrySet()) { + ret.add(wrapKey(e.getKey())); + } + return ret; + } + + @Override + public V put(K key, V value) { + V old = get(key); + map.put(unwrap(key), unwrap(value)); + return old; + } + + @Override + public void putAll(Map<? extends K, ? extends V> m) { + for (Map.Entry<?, ?> e : m.entrySet()) { + map.put(unwrap(e.getKey()), unwrap(e.getValue())); + } + } + + @Override + public V remove(Object key) { + return wrapValue(map.remove(unwrap(key))); + } + + @Override + public int size() { + return map.size(); + } + + @Override + public Collection<V> values() { + Collection<V> ret = new ArrayList<V>(); + for (Object v : map.values()) { + ret.add(wrapValue(v)); + } + return ret; + } + + } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/NumericFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/NumericFieldValue.java new file mode 100644 index 00000000000..776ca42c47d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/NumericFieldValue.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +public abstract class NumericFieldValue extends FieldValue { + + public abstract Number getNumber(); + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/PredicateFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/PredicateFieldValue.java new file mode 100644 index 00000000000..4978acc19d2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/PredicateFieldValue.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.predicate.Predicate; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlStream; + +import java.util.Objects; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class PredicateFieldValue extends FieldValue { + + private Predicate predicate; + + public PredicateFieldValue() { + this((Predicate)null); + } + + public PredicateFieldValue(Predicate predicate) { + this.predicate = predicate; + } + + public PredicateFieldValue(String predicateString) { + this(Predicate.fromString(predicateString)); + } + + public Predicate getPredicate() { + return predicate; + } + + public PredicateFieldValue setPredicate(Predicate predicate) { + this.predicate = predicate; + return this; + } + + @Override + public DataType getDataType() { + return DataType.PREDICATE; + } + + @Override + public void printXml(XmlStream xml) { + if (predicate == null) { + return; + } + xml.addContent(predicate.toString()); + } + + @Override + public void clear() { + predicate = null; + } + + @Override + public void assign(Object o) { + if (o == null) { + predicate = null; + } else if (o instanceof Predicate) { + predicate = (Predicate)o; + } else if (o instanceof PredicateFieldValue) { + predicate = ((PredicateFieldValue)o).predicate; + } else { + throw new IllegalArgumentException("Expected " + getClass().getName() + ", got " + + o.getClass().getName() + "."); + } + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public Object getWrappedValue() { + return predicate; + } + + @Override + public PredicateFieldValue clone() { + PredicateFieldValue obj = (PredicateFieldValue)super.clone(); + if (predicate != null) { + try { + obj.predicate = predicate.clone(); + } catch (CloneNotSupportedException e) { + throw new AssertionError(e); + } + } + return obj; + } + + @Override + public int hashCode() { + return predicate != null ? predicate.hashCode() : 31; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof PredicateFieldValue)) { + return false; + } + PredicateFieldValue rhs = (PredicateFieldValue)obj; + if (!Objects.equals(predicate, rhs.predicate)) { + return false; + } + return true; + } + + @Override + public String toString() { + return String.valueOf(predicate); + } + + public static PrimitiveDataType.Factory getFactory() { + return new PrimitiveDataType.Factory() { + + @Override + public FieldValue create() { + return new PredicateFieldValue(); + } + }; + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/Raw.java b/document/src/main/java/com/yahoo/document/datatypes/Raw.java new file mode 100644 index 00000000000..7d4d2430984 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/Raw.java @@ -0,0 +1,146 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * FieldValue which encapsulates a Raw value + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public final class Raw extends FieldValue { + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new Raw(); + } + } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 16, Raw.class); + private ByteBuffer value; + + public Raw() { + value = null; + } + + public Raw(ByteBuffer value) { + this.value = value; + } + + public Raw(byte [] buf) { + this.value = ByteBuffer.wrap(buf); + value.position(0); + } + + public ByteBuffer getByteBuffer() { + return value; + } + + @Override + public Raw clone() { + Raw raw = (Raw) super.clone(); + if (value.hasArray()) { + raw.value = ByteBuffer.wrap(Arrays.copyOf(value.array(), value.array().length)); + raw.value.position(value.position()); + } else { + byte[] copyBuf = new byte[value.capacity()]; + int origPos = value.position(); + value.position(0); + value.get(copyBuf); + value.position(origPos); + raw.value = ByteBuffer.wrap(copyBuf); + raw.value.position(value.position()); + } + return raw; + } + + @Override + public Object getWrappedValue() { + return value; + } + + @Override + public void clear() { + value = ByteBuffer.wrap(new byte[0]); + } + + @Override + public void assign(Object o) { + if (!checkAssign(o)) { + return; + } + if (o instanceof Raw) { + value = ((Raw) o).value; + } else if (o instanceof ByteBuffer) { + value = (ByteBuffer) o; + } else if (o instanceof byte[]) { + ByteBuffer byteBufVal = ByteBuffer.wrap((byte[]) o); + byteBufVal.position(0); + value = byteBufVal; + } else { + throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + @Override + public DataType getDataType() { + return DataType.RAW; + } + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printRawXml(this, xml); + } + + @Override + public String toString() { + ByteBuffer buf = value.slice(); + byte[] arr = new byte[buf.remaining()]; + buf.get(arr); + return com.yahoo.io.HexDump.toHexString(arr); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Raw)) return false; + if (!super.equals(o)) return false; + + Raw raw = (Raw) o; + + if (value != null ? !value.equals(raw.value) : raw.value != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (value != null ? value.hashCode() : 0); + return result; + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + /* (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java new file mode 100644 index 00000000000..29c5951f39b --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java @@ -0,0 +1,447 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.google.common.collect.ImmutableList; +import com.yahoo.collections.CollectionComparator; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.annotation.SpanTree; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * A StringFieldValue is a wrapper class that holds a String in {@link com.yahoo.document.Document}s and + * other {@link com.yahoo.document.datatypes.FieldValue}s. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class StringFieldValue extends FieldValue { + private static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new StringFieldValue(); + } + } + public static PrimitiveDataType.Factory getFactory() { return new Factory(); } + public static final int classId = registerClass(Ids.document + 15, StringFieldValue.class); + private String value; + private Map<String, SpanTree> spanTrees = null; + private static final boolean[] allowedAsciiChars = new boolean[0x80]; + + static { + allowedAsciiChars[0x0] = false; + allowedAsciiChars[0x1] = false; + allowedAsciiChars[0x2] = false; + allowedAsciiChars[0x3] = false; + allowedAsciiChars[0x4] = false; + allowedAsciiChars[0x5] = false; + allowedAsciiChars[0x6] = false; + allowedAsciiChars[0x7] = false; + allowedAsciiChars[0x8] = false; + allowedAsciiChars[0x9] = true; //tab + allowedAsciiChars[0xA] = true; //nl + allowedAsciiChars[0xB] = false; + allowedAsciiChars[0xC] = false; + allowedAsciiChars[0xD] = true; //cr + for (int i = 0xE; i < 0x20; i++) { + allowedAsciiChars[i] = false; + } + for (int i = 0x20; i < 0x7F; i++) { + allowedAsciiChars[i] = true; //printable ascii chars + } + allowedAsciiChars[0x7F] = true; //del - discouraged, but allowed + } + + + /** Creates a new StringFieldValue holding an empty String. */ + public StringFieldValue() { + value = ""; + } + + /** + * Creates a new StringFieldValue with the given value. + * + * @param value the value to wrap. + */ + public StringFieldValue(String value) { + if (value==null) throw new IllegalArgumentException("Value cannot be null"); + setValue(value); + } + + private void setValue(String value) { + for (int i = 0; i < value.length(); i++) { + char theChar = value.charAt(i); + int codePoint = value.codePointAt(i); + if (Character.isHighSurrogate(theChar)) { + //skip one char ahead, since codePointAt() consumes one more char in this case + ++i; + } + + //See http://www.w3.org/TR/2006/REC-xml11-20060816/#charsets + + if (codePoint < 0x80) { //ascii + if (allowedAsciiChars[codePoint]) { + continue; + } else { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + } + + //source cited above notes that 0x7F-0x84 and 0x86-0x9F are discouraged, but they are still allowed. + //see http://www.w3.org/International/questions/qa-controls + + if (codePoint < 0xFDD0) { + continue; + } + if (codePoint <= 0xFDDF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + + if (codePoint < 0x1FFFE) { + continue; + } + if (codePoint <= 0x1FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0x2FFFE) { + continue; + } + if (codePoint <= 0x2FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0x3FFFE) { + continue; + } + if (codePoint <= 0x3FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0x4FFFE) { + continue; + } + if (codePoint <= 0x4FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0x5FFFE) { + continue; + } + if (codePoint <= 0x5FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0x6FFFE) { + continue; + } + if (codePoint <= 0x6FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0x7FFFE) { + continue; + } + if (codePoint <= 0x7FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0x8FFFE) { + continue; + } + if (codePoint <= 0x8FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0x9FFFE) { + continue; + } + if (codePoint <= 0x9FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0xAFFFE) { + continue; + } + if (codePoint <= 0xAFFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0xBFFFE) { + continue; + } + if (codePoint <= 0xBFFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0xCFFFE) { + continue; + } + if (codePoint <= 0xCFFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0xDFFFE) { + continue; + } + if (codePoint <= 0xDFFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0xEFFFE) { + continue; + } + if (codePoint <= 0xEFFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0xFFFFE) { + continue; + } + if (codePoint <= 0xFFFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + if (codePoint < 0x10FFFE) { + continue; + } + if (codePoint <= 0x10FFFF) { + throw new IllegalArgumentException("StringFieldValue cannot contain code point 0x" + Integer.toHexString(codePoint).toUpperCase()); + } + } + this.value = value; + } + + /** + * Returns {@link com.yahoo.document.DataType}.STRING. + * + * @return DataType.STRING, always + * @see com.yahoo.document.DataType + */ + @Override + public DataType getDataType() { + return DataType.STRING; + } + + /** + * Clones this StringFieldValue and its span trees. + * + * @return a new deep-copied StringFieldValue + */ + @Override + public StringFieldValue clone() { + StringFieldValue strfval = (StringFieldValue) super.clone(); + if (spanTrees != null) { + strfval.spanTrees = new HashMap<String, SpanTree>(spanTrees.size()); + for (Map.Entry<String, SpanTree> entry : spanTrees.entrySet()) { + strfval.spanTrees.put(entry.getKey(), new SpanTree(entry.getValue())); + } + } + return strfval; + } + + /** Sets the wrapped String to be an empty String, and clears all span trees. */ + @Override + public void clear() { + value = ""; + if (spanTrees != null) { + spanTrees.clear(); + spanTrees = null; + } + } + + /** + * Sets a new value for this StringFieldValue. NOTE that doing so will clear all span trees from this value, + * since they most certainly will not make sense for a new string value. + * + * @param o the new String to assign to this. An argument of null is equal to calling clear(). + */ + @Override + public void assign(Object o) { + if (spanTrees != null) { + spanTrees.clear(); + spanTrees = null; + } + + if (!checkAssign(o)) { + return; + } + if (o instanceof StringFieldValue) { + spanTrees=((StringFieldValue)o).spanTrees; + } + if (o instanceof String) { + setValue((String) o); + } else if (o instanceof StringFieldValue || o instanceof NumericFieldValue) { + setValue(o.toString()); + } else { + throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + /** + * Returns an unmodifiable Collection of the span trees with annotations over this String, if any. + * + * @return an unmodifiable Collection of the span trees with annotations over this String, or an empty Collection + */ + public Collection<SpanTree> getSpanTrees() { + if (spanTrees == null) { + return ImmutableList.of(); + } + return ImmutableList.copyOf(spanTrees.values()); + } + + /** + * + * @return The map of spantrees. Might be null. + */ + public final Map<String, SpanTree> getSpanTreeMap() { + return spanTrees; + } + + /** + * Returns the span tree associated with the given name, or null if this does not exist. + * + * @param name the name of the span tree to return + * @return the span tree associated with the given name, or null if this does not exist. + */ + public SpanTree getSpanTree(String name) { + if (spanTrees == null) { + return null; + } + return spanTrees.get(name); + } + + /** + * Sets the span tree with annotations over this String. + * + * @param spanTree the span tree with annotations over this String + * @return the input spanTree for chaining + * @throws IllegalArgumentException if a span tree with the given name already exists. + */ + public SpanTree setSpanTree(SpanTree spanTree) { + if (spanTrees == null) { + spanTrees = new HashMap(1); + } + if (spanTrees.containsKey(spanTree.getName())) { + throw new IllegalArgumentException("Span tree " + spanTree.getName() + " already exists."); + } + spanTrees.put(spanTree.getName(), spanTree); + spanTree.setStringFieldValue(this); + return spanTree; + } + + /** + * Removes the span tree associated with the given name. + * + * @param name the name of the span tree to remove + * @return the span tree previously associated with the given name, or null if it did not exist. + */ + public SpanTree removeSpanTree(String name) { + if (spanTrees == null) { + return null; + } + SpanTree tree = spanTrees.remove(name); + if (tree != null) { + tree.setStringFieldValue(null); + } + return tree; + } + + /** + * Returns the String value wrapped by this StringFieldValue. + * + * @return the String value wrapped by this StringFieldValue. + */ + public String getString() { + return value; + } + + /** + * Returns the String value wrapped by this StringFieldValue. + * + * @return the String value wrapped by this StringFieldValue. + */ + @Override + public Object getWrappedValue() { + return value; + } + + /** + * Prints XML in Vespa Document XML format for this StringFieldValue. + * + * @param xml the stream to print to. + */ + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printStringXml(this, xml); + //TODO: add spanTree printing + } + + /** + * Returns the String value wrapped by this StringFieldValue. + * + * @return the String value wrapped by this StringFieldValue. + */ + @Override + public String toString() { + return value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof StringFieldValue)) return false; + if (!super.equals(o)) return false; + StringFieldValue that = (StringFieldValue) o; + if ((spanTrees != null) ? !spanTrees.equals(that.spanTrees) : that.spanTrees != null) return false; + if ((value != null) ? !value.equals(that.value) : that.value != null) return false; + return true; + } + + @Override + public int hashCode() { + return (value != null) ? value.hashCode() : super.hashCode(); + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + + //types are equal, this must be of this type + StringFieldValue otherValue = (StringFieldValue) fieldValue; + comp = value.compareTo(otherValue.value); + + if (comp != 0) { + return comp; + } + + if (spanTrees == null) { + comp = (otherValue.spanTrees == null) ? 0 : -1; + } else { + if (otherValue.spanTrees == null) { + comp = 1; + } else { + comp = CollectionComparator.compare(spanTrees.keySet(), otherValue.spanTrees.keySet()); + if (comp != 0) { + return comp; + } + comp = CollectionComparator.compare(spanTrees.values(), otherValue.spanTrees.values()); + } + } + return comp; + } + + /** + * Only for use by deserializer to avoid the cost of verifying input. + */ + public void setUnChecked(String s) { + value = s; + } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java new file mode 100644 index 00000000000..5a01dc33aa1 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -0,0 +1,391 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.collections.Hashlet; +import com.yahoo.document.*; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.vespa.objects.Ids; + +import java.util.*; + +/** + * Date: Apr 15, 2008 + * + * @author humbe + */ +public class Struct extends StructuredFieldValue { + + public static final int classId = registerClass(Ids.document + 33, Struct.class); + private Hashlet<Integer, FieldValue> values = new Hashlet<>(); + private int [] order = null; + + private int version; + + private int [] getInOrder() { + if (order == null) { + order = new int[values.size()]; + for (int i = 0; i < values.size(); i++) { + order[i] = values.key(i); + } + Arrays.sort(order); + } + return order; + } + + private void invalidateOrder() { + order = null; + } + + public Struct(DataType type) { + super((StructDataType) type); + this.version = Document.SERIALIZED_VERSION; + } + + @Override + public StructDataType getDataType() { + return (StructDataType)super.getDataType(); + } + + public void setVersion(int version) { + this.version = version; + } + + public int getVersion() { + return this.version; + } + + public com.yahoo.compress.CompressionType getCompressionType() { + if (getDataType().getCompressionConfig() == null) { + return com.yahoo.compress.CompressionType.NONE; + } + return getDataType().getCompressionConfig().type; + } + + public int getCompressionLevel() { + if ( getDataType().getCompressionConfig() == null) { + return 9; + } + return getDataType().getCompressionConfig().compressionLevel; + } + + public float getCompressionThreshold() { + if (getDataType().getCompressionConfig() == null) { + return .95f; + } + return getDataType().getCompressionConfig().threshold; + } + + @Override + public Struct clone() { + Struct struct = (Struct) super.clone(); + struct.values = new Hashlet<>(); + struct.values.reserve(values.size()); + for (int i = 0; i < values.size(); i++) { + struct.values.put(values.key(i), values.value(i).clone()); + } + return struct; + } + + @Override + public void clear() { + values = new Hashlet<>(); + invalidateOrder(); + } + + @Override + public Iterator<Map.Entry<Field, FieldValue>> iterator() { + return new FieldSet().iterator(); + } + + public Set<Map.Entry<Field, FieldValue>> getFields() { + return new FieldSet(); + } + + @Override + public void printXml(XmlStream xml) { + if (getDataType().equals(PositionDataType.INSTANCE)) { + try { + PositionDataType.renderXml(this, xml); + return; + } catch (RuntimeException e) { + // fallthrough to handling below + } + } + XmlSerializationHelper.printStructXml(this, xml); + } + + @Override + public FieldValue getFieldValue(Field field) { + return values.get(field.getId()); + } + + + @Override + public Field getField(String fieldName) { + return getDataType().getField(fieldName); + } + + @Override + public int getFieldCount() { + return values.size(); + } + + @Override + protected void doSetFieldValue(Field field, FieldValue value) { + if (field == null) { + throw new IllegalArgumentException("Invalid null field pointer"); + } + Field myField = getDataType().getField(field.getId()); + if (myField==null) { + throw new IllegalArgumentException("No such field in "+getDataType()+" : "+field.getName()); + } + if (!myField + .getDataType().isValueCompatible(value)) { + throw new IllegalArgumentException( + "Incompatible data types. Got " + value.getDataType() + + ", expected " + + myField.getDataType()); + } + + if (myField.getId() + != field.getId()) { + throw new IllegalArgumentException( + "Inconsistent field: " + field); + } + + int index = values.getIndexOfKey(field.getId()); + if (index == -1) { + values.put(field.getId(), value); + invalidateOrder(); + } else { + values.setValue(index, value); + } + } + + @Override + public FieldValue removeFieldValue(Field field) { + FieldValue found = values.get(field.getId()); + if (found != null) { + Hashlet<Integer, FieldValue> copy = new Hashlet<>(); + copy.reserve(values.size() - 1); + for (int i=0; i < values.size(); i++) { + if (values.key(i) != field.getId()) { + copy.put(values.key(i), values.value(i)); + } + } + values = copy; + invalidateOrder(); + } + return found; + } + + @Override + public void assign(Object o) { + if ((o instanceof Struct) && ((Struct) o).getDataType().equals(getDataType())) { + clear(); + Iterator<Map.Entry<Field,FieldValue>> otherValues = ((Struct) o).iterator(); + while (otherValues.hasNext()) { + Map.Entry<Field, FieldValue> otherEntry = otherValues.next(); + setFieldValue(otherEntry.getKey(), otherEntry.getValue()); + } + } else { + throw new IllegalArgumentException("Type " + o.getClass() + " can not specify a " + getClass() + " instance"); + } + } + + /** + * Clears this and assigns from the given {@link StructuredFieldValue} + */ + public void assignFrom(StructuredFieldValue sfv) { + clear(); + Iterator<Map.Entry<Field,FieldValue>> otherValues = sfv.iterator(); + while (otherValues.hasNext()) { + Map.Entry<Field, FieldValue> otherEntry = otherValues.next(); + setFieldValue(otherEntry.getKey(), otherEntry.getValue()); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Struct)) return false; + if (!super.equals(o)) return false; + + Struct struct = (Struct) o; + + return values.equals(struct.values); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + values.hashCode(); + return result; + } + + @Override + public String toString() { + StringBuilder retVal = new StringBuilder(); + retVal.append("Struct (").append(getDataType()).append("): "); + int [] increasing = getInOrder(); + for (int i = 0; i < increasing.length; i++) { + int id = increasing[i]; + retVal.append(getDataType().getField(id)).append("=").append(values.get(id)).append(", "); + } + return retVal.toString(); + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + @Override + public int compareTo(FieldValue obj) { + int cmp = super.compareTo(obj); + if (cmp != 0) { + return cmp; + } + Struct rhs = (Struct)obj; + cmp = values.size() - rhs.values.size(); + if (cmp != 0) { + return cmp; + } + StructDataType type = getDataType(); + for (Field field : type.getFields()) { + FieldValue lhsField = getFieldValue(field); + FieldValue rhsField = rhs.getFieldValue(field); + if (lhsField != null && rhsField != null) { + cmp = lhsField.compareTo(rhsField); + if (cmp != 0) { + return cmp; + } + } else if (lhsField != null || rhsField != null) { + return (lhsField != null ? -1 : 1); + } + } + return 0; + } + + /* + * (non-Javadoc) + * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader) + */ + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + private class FieldEntry implements Map.Entry<Field, FieldValue> { + private int id; + + private FieldEntry(int id) { + this.id = id; + } + + public Field getKey() { + return getDataType().getField(id); + } + + public FieldValue getValue() { + return values.get(id); + } + + public FieldValue setValue(FieldValue value) { + if (value == null) { + throw new NullPointerException("Null values in Struct not supported, use removeFieldValue() to remove value instead."); + } + + int index = values.getIndexOfKey(id); + FieldValue retVal = null; + if (index == -1) { + values.put(id, value); + invalidateOrder(); + } else { + retVal = values.value(index); + values.setValue(index, value); + } + + return retVal; + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof FieldEntry)) return false; + + FieldEntry that = (FieldEntry) o; + return (id == that.id); + } + + public int hashCode() { + return id; + } + } + + private class FieldSet extends AbstractSet<Map.Entry<Field, FieldValue>> { + @Override + public int size() { + return values.size(); + } + + @Override + public Iterator<Map.Entry<Field, FieldValue>> iterator() { + return new FieldSetIterator(); + } + + + } + + private class FieldSetIterator implements Iterator<Map.Entry<Field, FieldValue>> { + private int position = 0; + private int [] increasing = getInOrder(); + + public boolean hasNext() { + return (position < increasing.length); + } + + public Map.Entry<Field, FieldValue> next() { + if (position >= increasing.length) { + throw new NoSuchElementException("No more elements in collection"); + } + FieldEntry retval = new FieldEntry(increasing[position]); + position++; + return retval; + } + + public void remove() { + throw new UnsupportedOperationException("The set of fields and values of this struct is unmodifiable when accessed through this method."); + } + } + + public static <T> T getFieldValue(FieldValue struct, DataType structType, String fieldName, Class<T> fieldType) { + if (!(struct instanceof Struct)) { + return null; + } + if (!struct.getDataType().equals(structType)) { + return null; + } + FieldValue fieldValue = ((Struct)struct).getFieldValue(fieldName); + if (!fieldType.isInstance(fieldValue)) { + return null; + } + return fieldType.cast(fieldValue); + } + + public static <T> T getFieldValue(FieldValue struct, DataType structType, Field field, Class<T> fieldType) { + if (!(struct instanceof Struct)) { + return null; + } + if (!struct.getDataType().equals(structType)) { + return null; + } + FieldValue fieldValue = ((Struct)struct).getFieldValue(field); + if (!fieldType.isInstance(fieldValue)) { + return null; + } + return fieldType.cast(fieldValue); + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java new file mode 100644 index 00000000000..b4585a2188d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java @@ -0,0 +1,235 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.*; +import com.yahoo.vespa.objects.Ids; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public abstract class StructuredFieldValue extends CompositeFieldValue { + + public static final int classId = registerClass(Ids.document + 32, StructuredFieldValue.class); + + protected StructuredFieldValue(StructuredDataType type) { + super(type); + } + + @Override + public StructuredDataType getDataType() { + return (StructuredDataType)super.getDataType(); + } + + /** + * Returns the named field object, or null if that field does not exist. + * + * @param fieldName The name of the field to return. + * @return The corresponding field, or null. + */ + public abstract Field getField(String fieldName); + + /** + * Returns the value of the given field. If the field does not exist, this method returns null. + * + * @param field The field whose value to return. + * @return The value of the field, or null. + */ + public abstract FieldValue getFieldValue(Field field); + + /** + * Convenience method to return the value of a named field. This is the same as calling {@link #getField(String)}, + * and using the returned value to call {@link #getFieldValue(Field)}. If the named field does not exist, this + * method returns null. + * + * @param fieldName The name of the field whose value to return. + * @return The value of the field, or null. + */ + public FieldValue getFieldValue(String fieldName) { + Field field = getField(fieldName); + if (field == null) { + return null; + } + return getFieldValue(field); + } + + /** + * Sets the value of the given field. The type of the value must match the type of this field, i.e. + * <pre>field.getDataType().getValueClass().isAssignableFrom(value.getClass())</pre> must be true. + * + * @param field The field whose value to set. + * @param value The value to set. + * @return The previous value of the field, or null. + * @throws IllegalArgumentException If the value is not compatible with the field. + */ + public FieldValue setFieldValue(Field field, FieldValue value) { + if (value == null) { + return removeFieldValue(field); + } + DataType type = field.getDataType(); + if (!type.getValueClass().isAssignableFrom(value.getClass())) { + FieldValue tmp = type.createFieldValue(); + tmp.assign(value); + value = tmp; + } + FieldValue ret = getFieldValue(field); + doSetFieldValue(field, value); + return ret; + } + + protected abstract void doSetFieldValue(Field field, FieldValue value); + + /** + * Convenience method to set the value of a named field. This is the same as calling {@link #getField(String)}, and + * using the returned value to call {@link #setFieldValue(Field, FieldValue)}. If the named field does not exist, + * this method returns null. + * + * @param fieldName The name of the field whose value to set. + * @param value The value to set. + * @return The previous value of the field, or null. + */ + public FieldValue setFieldValue(String fieldName, FieldValue value) { + Field field = getField(fieldName); + if (field == null) { + return null; + } + return setFieldValue(field, value); + } + + public final FieldValue setFieldValue(Field field, String value) { + return setFieldValue(field, new StringFieldValue(value)); + } + + public final FieldValue setFieldValue(Field field, Double value) { + return setFieldValue(field, new DoubleFieldValue(value)); + } + + public final FieldValue setFieldValue(Field field, Integer value) { + return setFieldValue(field, new IntegerFieldValue(value)); + } + + public final FieldValue setFieldValue(Field field, Long value) { + return setFieldValue(field, new LongFieldValue(value)); + } + + public final FieldValue setFieldValue(Field field, Byte value) { + return setFieldValue(field, new ByteFieldValue(value)); + } + + public final FieldValue setFieldValue(String field, String value) { + return setFieldValue(field, new StringFieldValue(value)); + } + + public final FieldValue setFieldValue(String field, Double value) { + return setFieldValue(field, new DoubleFieldValue(value)); + } + + public final FieldValue setFieldValue(String field, Integer value) { + return setFieldValue(field, new IntegerFieldValue(value)); + } + + public final FieldValue setFieldValue(String field, Long value) { + return setFieldValue(field, new LongFieldValue(value)); + } + + public final FieldValue setFieldValue(String field, Byte value) { + return setFieldValue(field, new ByteFieldValue(value)); + } + /** + * Removes and returns a field value. + * + * @param field The field whose value to remove. + * @return The previous value of the field, or null. + */ + public abstract FieldValue removeFieldValue(Field field); + + /** + * Convenience method to remove the value of a named field. This is the same as calling {@link #getField(String)}, + * and using the returned value to call {@link #removeFieldValue(Field)}. If the named field does not exist, this + * method returns null. + * + * @param fieldName The name of the field whose value to remove. + * @return The previous value of the field, or null. + */ + public FieldValue removeFieldValue(String fieldName) { + Field field = getField(fieldName); + if (field == null) { + return null; + } + return removeFieldValue(field); + } + + public abstract void clear(); + + public abstract int getFieldCount(); + + public abstract Iterator<Map.Entry<Field, FieldValue>> iterator(); + + @Override + public FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, + FieldPathIteratorHandler handler) { + if (pos < fieldPath.size()) { + if (fieldPath.get(pos).getType() == FieldPathEntry.Type.STRUCT_FIELD) { + FieldValue fieldVal = getFieldValue(fieldPath.get(pos).getFieldRef()); + if (fieldVal != null) { + FieldPathIteratorHandler.ModificationStatus status = fieldVal.iterateNested(fieldPath, pos + 1, handler); + if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) { + removeFieldValue(fieldPath.get(pos).getFieldRef()); + return FieldPathIteratorHandler.ModificationStatus.MODIFIED; + } else { + if (isGenerated()) { + // If this is a generated doc, the operations on the FieldValue in iterateNested do not write through to the doc, + // so set the field again here. Should be a cleaner way to do this. + setFieldValue(fieldPath.get(pos).getFieldRef(), fieldVal); + } + return status; + } + } else if (handler.createMissingPath()) { + FieldValue newVal = fieldPath.get(pos).getFieldRef().getDataType().createFieldValue(); + FieldPathIteratorHandler.ModificationStatus status = newVal.iterateNested(fieldPath, pos + 1, handler); + if (status == FieldPathIteratorHandler.ModificationStatus.MODIFIED) { + setFieldValue(fieldPath.get(pos).getFieldRef(), newVal); + return status; + } + } + return FieldPathIteratorHandler.ModificationStatus.NOT_MODIFIED; + } + throw new IllegalArgumentException("Illegal field path " + fieldPath.get(pos) + " for struct value"); + } else { + FieldPathIteratorHandler.ModificationStatus status = handler.modify(this); + if (status == FieldPathIteratorHandler.ModificationStatus.REMOVED) { + return status; + } + if (handler.onComplex(this)) { + List<Field> fieldsToRemove = new ArrayList<Field>(); + for (Iterator<Map.Entry<Field, FieldValue>> iter = iterator(); iter.hasNext();) { + Map.Entry<Field, FieldValue> entry = iter.next(); + FieldPathIteratorHandler.ModificationStatus currStatus = entry.getValue().iterateNested(fieldPath, pos, handler); + if (currStatus == FieldPathIteratorHandler.ModificationStatus.REMOVED) { + fieldsToRemove.add(entry.getKey()); + status = FieldPathIteratorHandler.ModificationStatus.MODIFIED; + } else if (currStatus == FieldPathIteratorHandler.ModificationStatus.MODIFIED) { + status = currStatus; + } + } + for (Field field : fieldsToRemove) { + removeFieldValue(field); + } + } + return status; + } + } + + /** + * Generated Document subclasses should override this and return true. This is used instead of using class.getAnnotation(Generated.class), because that is so slow. + * @return true if in a concrete subtype of Document + */ + protected boolean isGenerated() { + return false; + } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java new file mode 100644 index 00000000000..82e3ff36c1c --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlStream; +import com.yahoo.tensor.Tensor; + +import java.util.Objects; +import java.util.Optional; + +/** + * Field value class that wraps a tensor. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class TensorFieldValue extends FieldValue { + + private Optional<Tensor> tensor; + + public TensorFieldValue() { + tensor = Optional.empty(); + } + + public TensorFieldValue(Tensor tensor) { + this.tensor = Optional.of(tensor); + } + + public Optional<Tensor> getTensor() { + return tensor; + } + + @Override + public DataType getDataType() { + return DataType.TENSOR; + } + + @Override + public void printXml(XmlStream xml) { + // TODO (geirst) + } + + @Override + public void clear() { + tensor = Optional.empty(); + } + + @Override + public void assign(Object o) { + if (o == null) { + tensor = Optional.empty(); + } else if (o instanceof Tensor) { + tensor = Optional.of((Tensor)o); + } else if (o instanceof TensorFieldValue) { + tensor = ((TensorFieldValue)o).getTensor(); + } else { + throw new IllegalArgumentException("Expected class '" + getClass().getName() + "', got '" + + o.getClass().getName() + "'."); + } + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TensorFieldValue)) { + return false; + } + TensorFieldValue rhs = (TensorFieldValue)o; + if (!Objects.equals(tensor, rhs.tensor)) { + return false; + } + return true; + } + + public static PrimitiveDataType.Factory getFactory() { + return new PrimitiveDataType.Factory() { + + @Override + public FieldValue create() { + return new TensorFieldValue(); + } + }; + } +} + diff --git a/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java new file mode 100644 index 00000000000..a1e0b12b6a0 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/UriFieldValue.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.net.Url; + +import java.net.URI; + +/** + * Created with IntelliJ IDEA. + * User: magnarn + * Date: 11/2/12 + * Time: 2:37 PM + */ +public class UriFieldValue extends StringFieldValue { + public static class Factory extends PrimitiveDataType.Factory { + public FieldValue create() { + return new UriFieldValue(); + } + } + public UriFieldValue() { super(); } + + public UriFieldValue(String value) { + super(value); + Url.fromString(value); // Throws if value is invalid. + } + + @Override + public void assign(Object obj) { + if (obj instanceof URI) { + obj = obj.toString(); + } + super.assign(obj); + } + + @Override + public DataType getDataType() { + return DataType.URI; + } + + @Override + public void deserialize(Field field, FieldReader reader) { + super.deserialize(field, reader); + Url.fromString(toString()); // Throws if value is invalid. + } +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/WeightedSet.java b/document/src/main/java/com/yahoo/document/datatypes/WeightedSet.java new file mode 100644 index 00000000000..5e56f247a7f --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/WeightedSet.java @@ -0,0 +1,418 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.datatypes; + +import com.yahoo.collections.CollectionComparator; +import com.yahoo.document.*; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlSerializationHelper; +import com.yahoo.document.serialization.XmlStream; + +import java.util.*; + +/** + * A weighted set, a unique set of keys with an associated integer weight. This class + * uses an encapsulated Map (actually a LinkedHashMap) that associates each key + * with its weight (value). + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public final class WeightedSet<K extends FieldValue> extends CollectionFieldValue<K> implements Map<K, Integer> { + + private MapFieldValue<K, IntegerFieldValue> map; + + /** + * Creates a new WeightedSet. + * + * @param type the data type for the field that this weighted set is associated with + */ + public WeightedSet(DataType type) { + this(type, 1); + } + + /** + * Creates a new weighted set with a given initial capacity. + * + * @param initialCapacity the initial capacity to use for the encapsulated Map + */ + public WeightedSet(DataType type, int initialCapacity) { + super((WeightedSetDataType) type); + clearAndReserve(initialCapacity); + } + + @Override + public WeightedSetDataType getDataType() { + return (WeightedSetDataType) super.getDataType(); + } + + @Override + public Iterator<K> fieldValueIterator() { + return map.keySet().iterator(); + } + + @Override + public void assign(Object o) { + if (!checkAssign(o)) { + return; + } + + if (o instanceof WeightedSet) { + WeightedSet wset = (WeightedSet) o; + if (getDataType().equals(wset.getDataType())) { + map.assign(wset.map); + } else { + throw new IllegalArgumentException("Cannot assign a weighted set of type " + wset.getDataType() + + " to a weighted set of type " + getDataType()); + } + } else if (o instanceof Map) { + map = new WeightedSetWrapper((Map)o, map.getDataType()); + } else { + throw new IllegalArgumentException("Class " + o.getClass() + " not applicable to an " + this.getClass() + " instance."); + } + } + + @Override + public WeightedSet clone() { + WeightedSet<K> newSet = (WeightedSet<K>) super.clone(); + newSet.map = (MapFieldValue<K, IntegerFieldValue>) map.clone(); + return newSet; + } + + + @Override + public void printXml(XmlStream xml) { + XmlSerializationHelper.printWeightedSetXml(this, xml); + } + + /** + * Returns the number of key-weight pairs in this set. + * + * @return the number of key-weight pairs in this set + */ + public int size() { + return map.size(); + } + + public boolean add(K value) { + put(value, 1); + return true; + } + + @Override + public Object getWrappedValue() { + if (map instanceof WeightedSet.WeightedSetWrapper) { + return ((WeightedSet.WeightedSetWrapper) map).map; + } + return map.getWrappedValue(); + } + + @Override + public boolean contains(Object o) { + return map.keySet().contains(o); + } + + /** + * Checks if this set is empty. + * + * @return true if the set is empty + */ + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public Iterator<K> iterator() { + return map.keySet().iterator(); + } + + @Override + public boolean removeValue(FieldValue o) { + return super.removeValue(o, map.keySet()); + } + + /** + * Checks whether this set contains the specified key. + * + * @param key the key to search for + * @return true if this set contains this key + */ + public boolean containsKey(Object key) { + return map.containsKey(key); + } + + public boolean containsValue(Object value) { + return map.containsValue(value); + } + + /** + * Returns the weight associated with the specified key. + * + * @param key the key to return the weight for + * @return the weight associated with the specified key, or null (if not found) + */ + public Integer get(Object key) { + if (!(key instanceof FieldValue)) { + throw new IllegalArgumentException("Only FieldValues are allowed as keys."); + } + IntegerFieldValue ifv = map.get(key); + return ifv != null ? ifv.getInteger() : null; + } + + /** + * Add a key with an associated weight to this set. If the key is already present in this set, the previous + * association is replaced. Checks to validate that all keys are of the same type. + * + * @param key the key to add + * @param weight the weight to associate with this key + * @return the weight that was previously associated with this key, or null (if there was no previous key) + */ + public Integer put(K key, Integer weight) { + verifyElementCompatibility(key); + IntegerFieldValue ifv = putUnChecked(key, new IntegerFieldValue(weight)); + return ifv != null ? ifv.getInteger() : null; + } + + /** + * Add a key with an associated weight to this set. If the key is already present in this set, the previous + * association is replaced. + * + * @param key the key to add + * @param weight the weight to associate with this key + * @return the weight that was previously associated with this key, or null (if there was no previous key) + */ + public IntegerFieldValue putUnChecked(K key, IntegerFieldValue weight) { + return map.put(key, weight); + } + + /** + * Remove a key-weight association from this set. + * + * @param key the key to remove + * @return the weight that was previously associated with this key, or null (if there was no previous key) + */ + public Integer remove(Object key) { + IntegerFieldValue ifv = map.remove(key); + return ifv != null ? ifv.getInteger() : null; + } + + public void putAll(Map<? extends K, ? extends Integer> t) { + for (Entry<? extends K, ? extends Integer> entry : t.entrySet()) { + put(entry.getKey(), entry.getValue()); + } + } + + /** Remove all key-weight associations in this set. */ + public void clear() { + map.clear(); + } + + /** + * Reserve space for this amount of keys in order to avoid resizing + */ + public void clearAndReserve(int count) { + map = new MapFieldValue<>(new MapDataType(getDataType().getNestedType(), DataType.INT), count); + } + + Map<K, Integer> getPrimitiveMap() { + Map<K, Integer> retVal = new LinkedHashMap<>(); + for (Entry<K, IntegerFieldValue> entry : map.entrySet()) { + retVal.put(entry.getKey(), entry.getValue().getInteger()); + } + return retVal; + } + + public Collection<Integer> values() { + return getPrimitiveMap().values(); + } + + public Set<K> keySet() { + return map.keySet(); + } + + public Set<Entry<K, Integer>> entrySet() { + return getPrimitiveMap().entrySet(); + } + + /** + * Checks if another object is equal to this set. + * + * @param o the object to check for equality with + * @return true if o is an instance of WeightedSet and the two encapsulated Maps are equal, false otherwise + */ + public boolean equals(Object o) { + if (!(o instanceof WeightedSet)) return false; + WeightedSet otherSet = (WeightedSet) o; + return (super.equals(o) && map.equals(otherSet.map)); + } + + /** + * Uses hashCode() from the encapsulated Map. + * + * @return the hash code of this set + */ + public int hashCode() { + return map.hashCode(); + } + + /** + * Uses toString() from the encapsulated Map. + * + * @return the toString() of this set + */ + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("WeightedSet(").append(getDataType()); + for (Map.Entry entry : map.entrySet()) { + sb.append("\n key: ").append(entry.getKey().getClass()).append(": ").append(entry.getKey()); + sb.append("\n value: ").append(entry.getValue().getClass()).append(": ").append(entry.getValue()); + } + return sb.append("\n)").toString(); + } + + @Override + public void serialize(Field field, FieldWriter writer) { + writer.write(field, this); + } + + @Override + public void deserialize(Field field, FieldReader reader) { + reader.read(field, this); + } + + @Override + FieldPathIteratorHandler.ModificationStatus iterateNested(FieldPath fieldPath, int pos, FieldPathIteratorHandler handler) { + FieldPathIteratorHandler.ModificationStatus status = map.iterateNested(fieldPath, pos, handler, this); + return status; + } + + @Override + public int compareTo(FieldValue fieldValue) { + int comp = super.compareTo(fieldValue); + + if (comp != 0) { + return comp; + } + + //types are equal, this must be of this type + WeightedSet otherValue = (WeightedSet) fieldValue; + comp = CollectionComparator.compare(map.keySet(), otherValue.map.keySet()); + + if (comp != 0) { + return comp; + } + + return CollectionComparator.compare(map.values(), otherValue.map.values()); + } + + + /** + * Weighted set MapFieldValue, backed by map of native Java types. + * Note: The key type of this is FieldValue, not K. + * @author vegardh + * + */ + class WeightedSetWrapper extends MapFieldValue<K, IntegerFieldValue> { + Map<Object, Integer> map = new HashMap<Object, Integer>(); + private DataType keyTypeVespa = getDataType().getKeyType(); + private DataType valTypeVespa = DataType.INT; + + public WeightedSetWrapper(Map map, MapDataType dt) { + super(dt); + this.map=map; + } + + private Object unwrap(Object o) { + return (o instanceof FieldValue ? ((FieldValue) o).getWrappedValue() : o); + } + + @SuppressWarnings("unchecked") + private K wrapKey(Object o) { + if (o==null) return null; + return (K) keyTypeVespa.createFieldValue(o); + } + + private IntegerFieldValue wrapValue(Object o) { + if (o==null) return null; + return (IntegerFieldValue)valTypeVespa.createFieldValue(o); + } + + @Override + public void clear() { + map.clear(); + } + + @Override + public boolean containsKey(Object key) { + return map.containsKey(unwrap(key)); + } + + @Override + public boolean containsValue(Object value) { + return map.containsValue(unwrap(value)); + } + + @Override + public Set<java.util.Map.Entry<K, IntegerFieldValue>> entrySet() { + Map<K, IntegerFieldValue> ret = new HashMap<>(); + for (Map.Entry e : map.entrySet()) { + ret.put(wrapKey(e.getKey()), wrapValue(e.getValue())); + } + return ret.entrySet(); + } + + @Override + public IntegerFieldValue get(Object key) { + Object o = map.get(unwrap(key)); + return o == null ? null : wrapValue(o); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + + @Override + public Set<K> keySet() { + Set<K> ret = new HashSet<>(); + for (Map.Entry e : map.entrySet()) { + ret.add(wrapKey(e.getKey())); + } + return ret; + } + + @Override + public IntegerFieldValue put(FieldValue key, IntegerFieldValue value) { + IntegerFieldValue old = get(key); + map.put(unwrap(key), (Integer) unwrap(value)); + return old; + } + + @Override + public void putAll(Map<? extends K, ? extends IntegerFieldValue> m) { + for (Map.Entry<?, ?> e : m.entrySet()) { + map.put(unwrap(e.getKey()), (Integer) unwrap(e.getValue())); + } + } + + @Override + public IntegerFieldValue remove(Object key) { + return wrapValue(map.remove(unwrap(key))); + } + + @Override + public int size() { + return map.size(); + } + + @Override + public Collection<IntegerFieldValue> values() { + Collection<IntegerFieldValue> ret = new ArrayList<>(); + for (Object v : map.values()) { + ret.add(wrapValue(v)); + } + return ret; + } + + } + +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/package-info.java b/document/src/main/java/com/yahoo/document/datatypes/package-info.java new file mode 100644 index 00000000000..67ed3c15393 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/datatypes/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.datatypes; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/declaration/.gitignore b/document/src/main/java/com/yahoo/document/declaration/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/declaration/.gitignore diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java new file mode 100644 index 00000000000..28673adbd88 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldpathupdate; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldPathIteratorHandler; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.serialization.DocumentUpdateReader; +import com.yahoo.document.serialization.VespaDocumentSerializerHead; + +/** + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class AddFieldPathUpdate extends FieldPathUpdate { + class IteratorHandler extends FieldPathIteratorHandler { + Array newValues; + + IteratorHandler(Array newValues) { + this.newValues = newValues; + } + + @SuppressWarnings({ "unchecked" }) + @Override + public ModificationStatus doModify(FieldValue fv) { + for (Object newValue : newValues.getValues()) { + ((CollectionFieldValue)fv).add((FieldValue) newValue); + } + return ModificationStatus.MODIFIED; + } + + @Override + public boolean createMissingPath() { + return true; + } + + @Override + public boolean onComplex(FieldValue fv) { + return false; + } + + public Array getNewValues() { + return newValues; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + IteratorHandler that = (IteratorHandler) o; + + if (!newValues.equals(that.newValues)) return false; + + return true; + } + + @Override + public int hashCode() { + return newValues.hashCode(); + } + } + + IteratorHandler handler; + + public AddFieldPathUpdate(DocumentType type, String fieldPath, String whereClause, Array newValues) { + super(FieldPathUpdate.Type.ADD, type, fieldPath, whereClause); + setNewValues(newValues); + } + + public AddFieldPathUpdate(DocumentType type, String fieldPath, Array newValues) { + super(FieldPathUpdate.Type.ADD, type, fieldPath, null); + setNewValues(newValues); + } + + public AddFieldPathUpdate(DocumentType type, DocumentUpdateReader reader) { + super(FieldPathUpdate.Type.ADD, type, reader); + reader.read(this); + } + + public void setNewValues(Array value) { + handler = new IteratorHandler(value); + } + + public Array getNewValues() { + return handler.getNewValues(); + } + + public FieldPathIteratorHandler getIteratorHandler(Document doc) { + return handler; + } + + @Override + public void serialize(VespaDocumentSerializerHead data) { + data.write(this); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + + AddFieldPathUpdate that = (AddFieldPathUpdate) o; + + if (!handler.equals(that.handler)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + handler.hashCode(); + return result; + } + + @Override + public String toString() { + return "Add: " + super.toString() + " : " + handler.getNewValues(); + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java new file mode 100644 index 00000000000..b799a56197f --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java @@ -0,0 +1,281 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldpathupdate; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentCalculator; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.FieldPathIteratorHandler; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.NumericFieldValue; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.serialization.DocumentUpdateReader; +import com.yahoo.document.serialization.VespaDocumentSerializerHead; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class AssignFieldPathUpdate extends FieldPathUpdate { + class SimpleAssignIteratorHandler extends FieldPathIteratorHandler { + FieldValue newValue; + boolean removeIfZero; + boolean createMissingPath; + + SimpleAssignIteratorHandler(FieldValue newValue, boolean removeIfZero, boolean createMissingPath) { + this.newValue = newValue; + this.removeIfZero = removeIfZero; + this.createMissingPath = createMissingPath; + } + + @Override + public ModificationStatus doModify(FieldValue fv) { + if (!fv.getDataType().equals(newValue.getDataType())) { + throw new IllegalArgumentException("Trying to assign " + newValue + " of type " + newValue.getDataType() + " to an instance of " + fv.getDataType()); + } else { + if (removeIfZero && (newValue instanceof NumericFieldValue) && ((NumericFieldValue)newValue).getNumber().longValue() == 0) { + return ModificationStatus.REMOVED; + } + fv.assign(newValue); + } + return ModificationStatus.MODIFIED; + } + + @Override + public boolean createMissingPath() { + return createMissingPath; + } + + @Override + public boolean onComplex(FieldValue fv) { + return false; + } + } + + class MathAssignIteratorHandler extends FieldPathIteratorHandler { + DocumentCalculator calc; + Document doc; + boolean removeIfZero; + boolean createMissingPath; + + MathAssignIteratorHandler(String expression, Document doc, boolean removeIfZero, boolean createMissingPath) throws ParseException { + this.calc = new DocumentCalculator(expression); + this.doc = doc; + this.removeIfZero = removeIfZero; + this.createMissingPath = createMissingPath; + } + + @Override + public ModificationStatus doModify(FieldValue fv) { + if (fv instanceof NumericFieldValue) { + Map<String, Object> vars = new HashMap<String, Object>(); + for (Map.Entry<String, IndexValue> entry : getVariables().entrySet()) { + if (entry.getValue().getKey() != null && entry.getValue().getKey() instanceof NumericFieldValue) { + vars.put(entry.getKey(), ((NumericFieldValue)entry.getValue().getKey()).getNumber()); + } else { + vars.put(entry.getKey(), entry.getValue().getIndex()); + } + } + vars.put("value", ((NumericFieldValue)fv).getNumber()); + + try { + Number d = calc.evaluate(doc, vars); + if (removeIfZero && d.longValue() == 0) { + return ModificationStatus.REMOVED; + } else { + fv.assign(calc.evaluate(doc, vars)); + } + } catch (IllegalArgumentException e) { + // Ignore divide by zero + return ModificationStatus.NOT_MODIFIED; + } + } else { + throw new IllegalArgumentException("Trying to perform arithmetic on " + fv + " of type " + fv.getDataType()); + } + return ModificationStatus.MODIFIED; + } + + @Override + public boolean createMissingPath() { + return createMissingPath; + } + + @Override + public boolean onComplex(FieldValue fv) { + return false; + } + } + + FieldValue fieldValue = null; + String expression = null; + boolean createMissingPath = true; + boolean removeIfZero = false; + + // Flag bits + public static final int ARITHMETIC_EXPRESSION = 1; + public static final int REMOVE_IF_ZERO = 2; + public static final int CREATE_MISSING_PATH = 4; + + /** + * Creates an assignment update that overwrites the old value with the given new value. + * + * @param type The document type the assignment works on. + * @param fieldPath The field path of the field to be overwritten. + * @param whereClause A document selection string that selects documents and variables to be updated. + * @param newValue The new value of the assignment. + */ + public AssignFieldPathUpdate(DocumentType type, String fieldPath, String whereClause, FieldValue newValue) { + super(FieldPathUpdate.Type.ASSIGN, type, fieldPath, whereClause); + setNewValue(newValue); + } + + public AssignFieldPathUpdate(DocumentType type, String fieldPath, FieldValue newValue) { + super(FieldPathUpdate.Type.ASSIGN, type, fieldPath, null); + setNewValue(newValue); + } + + /** + * Creates an assign statement based on a mathematical expression. + * + * @param type The document type the assignment works on. + * @param fieldPath The field path of the field to be overwritten. + * @param whereClause A document selection string that selects documents and variables to be updated. + * @param expression The mathematical expression to apply. Use $value to signify the previous value of the field. + */ + public AssignFieldPathUpdate(DocumentType type, String fieldPath, String whereClause, String expression) { + super(FieldPathUpdate.Type.ASSIGN, type, fieldPath, whereClause); + setExpression(expression); + } + + /** + * Creates an assign update from a serialized object. + * + * @param type The document type the assignment will work on. + * @param reader A reader that can deserialize something into this object. + */ + public AssignFieldPathUpdate(DocumentType type, DocumentUpdateReader reader) { + super(FieldPathUpdate.Type.ASSIGN, type, reader); + reader.read(this); + } + + /** + * Turns this assignment into a literal one. + * + * @param value The new value to assign to the document. + */ + public void setNewValue(FieldValue value) { + fieldValue = value; + expression = null; + } + + /** + * + * @return Returns the value to assign, or null if this is a mathematical expression. + */ + public FieldValue getNewValue() { + return fieldValue; + } + + /** + * Turns this assignment into a mathematical expression assignment. + * + * @param value The expression to use for assignment. + */ + public void setExpression(String value) { + expression = value; + fieldValue = null; + } + + /** + * + * @return Returns the arithmetic expression to assign, or null if this is not a mathematical expression. + */ + public String getExpression() { + return expression; + } + + /** + * If set to true, and the new value assigned evaluates to a numeric value of 0, removes the value instead of setting it. + * Default is false. + */ + public void setRemoveIfZero(boolean removeIfZero) { + this.removeIfZero = removeIfZero; + } + + /** + * If set to true, and any part of the field path specified does not exist (except for array indexes), we create the path as necessary. + * Default is true. + */ + public void setCreateMissingPath(boolean createMissingPath) { + this.createMissingPath = createMissingPath; + } + + /** + * + * @return Returns true if this assignment is an arithmetic operation. + */ + public boolean isArithmetic() { + return expression != null; + } + + FieldPathIteratorHandler getIteratorHandler(Document doc) { + if (expression != null) { + try { + return new MathAssignIteratorHandler(expression, doc, removeIfZero, createMissingPath); + } catch (ParseException e) { + return null; + } + } else { + return new SimpleAssignIteratorHandler(fieldValue, removeIfZero, createMissingPath); + } + } + + @Override + public void serialize(VespaDocumentSerializerHead data) { + data.write(this); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + + AssignFieldPathUpdate that = (AssignFieldPathUpdate) o; + + if (createMissingPath != that.createMissingPath) return false; + if (removeIfZero != that.removeIfZero) return false; + if (expression != null ? !expression.equals(that.expression) : that.expression != null) return false; + if (fieldValue != null ? !fieldValue.equals(that.fieldValue) : that.fieldValue != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (fieldValue != null ? fieldValue.hashCode() : 0); + result = 31 * result + (expression != null ? expression.hashCode() : 0); + result = 31 * result + (createMissingPath ? 1 : 0); + result = 31 * result + (removeIfZero ? 1 : 0); + return result; + } + + @Override + public String toString() { + return "Assign: " + super.toString() + " : " + (isArithmetic() ? getExpression() : getNewValue().toString()); + } + + public boolean getCreateMissingPath() { + return createMissingPath; + } + + public boolean getRemoveIfZero() { + return removeIfZero; + } + + public FieldValue getFieldValue() { + return fieldValue; + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java new file mode 100644 index 00000000000..318b696ce7a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldpathupdate; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.FieldPath; +import com.yahoo.document.datatypes.FieldPathIteratorHandler; +import com.yahoo.document.select.DocumentSelector; +import com.yahoo.document.select.Result; +import com.yahoo.document.select.ResultList; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.serialization.DocumentUpdateReader; +import com.yahoo.document.serialization.VespaDocumentSerializerHead; + +/** + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public abstract class FieldPathUpdate { + + public static enum Type { + ASSIGN(0), + REMOVE(1), + ADD(2); + + private final int code; + + private Type(int code) { + this.code = code; + } + + public static Type valueOf(int code) { + for (Type type : values()) { + if (type.code == code) { + return type; + } + } + throw new IllegalArgumentException("Field path update type " + code + " not supported."); + } + + public int getCode() { + return code; + } + } + + private FieldPath fieldPath; + private DocumentSelector selector; + private String originalFieldPath; + private String whereClause; + private Type updType; + private DocumentType docType; + + public FieldPathUpdate(Type updType, DocumentType docType, String fieldPath, String whereClause) { + this.updType = updType; + this.docType = docType; + + try { + setWhereClause(whereClause); + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + setFieldPath(fieldPath); + } + + public FieldPathUpdate(Type updType, DocumentType docType, DocumentUpdateReader reader) { + this.updType = updType; + this.docType = docType; + reader.read(this); + } + + public Type getUpdateType() { + return updType; + } + + public DocumentType getDocumentType() { + return docType; + } + + public void setFieldPath(String fieldPath) { + originalFieldPath = fieldPath; + this.fieldPath = docType.buildFieldPath(fieldPath); + } + + public FieldPath getFieldPath() { + return fieldPath; + } + + public String getOriginalFieldPath() { + return originalFieldPath; + } + + public void setWhereClause(String whereClause) throws ParseException { + this.whereClause = whereClause; + selector = null; + if (whereClause != null && !whereClause.equals("")) { + selector = new DocumentSelector(whereClause); + } + } + + public DocumentSelector getWhereClause() { + return selector; + } + + public String getOriginalWhereClause() { + return whereClause; + } + + public void applyTo(Document doc) { + if (selector == null) { + FieldPathIteratorHandler handler = getIteratorHandler(doc); + doc.iterateNested(fieldPath, 0, handler); + } else { + ResultList results = selector.getMatchingResultList(new DocumentPut(doc)); + + for (ResultList.ResultPair rp : results.getResults()) { + if (rp.getResult() == Result.TRUE) { + FieldPathIteratorHandler handler = getIteratorHandler(doc); + handler.getVariables().clear(); + handler.getVariables().putAll(rp.getVariables()); + + doc.iterateNested(fieldPath, 0, handler); + } + } + } + } + + public void serialize(VespaDocumentSerializerHead data) { + data.write(this); + } + + public static FieldPathUpdate create(Type type, DocumentType docType, DocumentUpdateReader reader) throws ParseException { + switch (type) { + case ASSIGN: + return new AssignFieldPathUpdate(docType, reader); + case ADD: + return new AddFieldPathUpdate(docType, reader); + case REMOVE: + return new RemoveFieldPathUpdate(docType, reader); + } + throw new IllegalArgumentException("Field path update type '" + type + "' not supported."); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FieldPathUpdate that = (FieldPathUpdate) o; + + if (docType != null ? !docType.equals(that.docType) : that.docType != null) return false; + if (originalFieldPath != null ? !originalFieldPath.equals(that.originalFieldPath) : that.originalFieldPath != null) + return false; + if (whereClause != null ? !whereClause.equals(that.whereClause) : that.whereClause != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = originalFieldPath != null ? originalFieldPath.hashCode() : 0; + result = 31 * result + (whereClause != null ? whereClause.hashCode() : 0); + result = 31 * result + (docType != null ? docType.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "fieldpath=\"" + originalFieldPath + "\"" + (whereClause != null ? " where=\"" + whereClause + "\"" : ""); + } + + abstract FieldPathIteratorHandler getIteratorHandler(Document doc); +} diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java new file mode 100644 index 00000000000..fc4a5b39f92 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/RemoveFieldPathUpdate.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldpathupdate; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.FieldPathIteratorHandler; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.serialization.DocumentUpdateReader; + +/** + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public class RemoveFieldPathUpdate extends FieldPathUpdate { + class IteratorHandler extends FieldPathIteratorHandler { + IteratorHandler() { + } + + @Override + public ModificationStatus doModify(FieldValue fv) { + return ModificationStatus.REMOVED; + } + + @Override + public boolean onComplex(FieldValue fv) { + return false; + } + } + + IteratorHandler handler; + + public RemoveFieldPathUpdate(DocumentType type, String fieldPath, String whereClause) { + super(FieldPathUpdate.Type.REMOVE, type, fieldPath, whereClause); + handler = new IteratorHandler(); + } + + public RemoveFieldPathUpdate(DocumentType type, String fieldPath) { + super(FieldPathUpdate.Type.REMOVE, type, fieldPath, null); + handler = new IteratorHandler(); + } + + public RemoveFieldPathUpdate(DocumentType type, DocumentUpdateReader reader) { + super(FieldPathUpdate.Type.REMOVE, type, reader); + reader.read(this); + handler = new IteratorHandler(); + } + + FieldPathIteratorHandler getIteratorHandler(Document doc) { + return handler; + } + + @Override + public String toString() { + return "Remove: " + super.toString(); + } +}
\ No newline at end of file diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/package-info.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/package-info.java new file mode 100644 index 00000000000..8e9f4069386 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.fieldpathupdate; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/fieldset/AllFields.java b/document/src/main/java/com/yahoo/document/fieldset/AllFields.java new file mode 100644 index 00000000000..3dff7c1e4e6 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/AllFields.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +/** + * Created with IntelliJ IDEA. + * User: thomasg + * Date: 4/25/12 + * Time: 3:18 PM + * To change this template use File | Settings | File Templates. + */ +public class AllFields implements FieldSet { + @Override + public boolean contains(FieldSet o) { + return true; + } + + @Override + public FieldSet clone() throws CloneNotSupportedException { + return new AllFields(); + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/BodyFields.java b/document/src/main/java/com/yahoo/document/fieldset/BodyFields.java new file mode 100644 index 00000000000..50a72fa2fde --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/BodyFields.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +import com.yahoo.document.Field; + +/** + * Created with IntelliJ IDEA. + * User: thomasg + * Date: 4/25/12 + * Time: 3:18 PM + * To change this template use File | Settings | File Templates. + */ +public class BodyFields implements FieldSet { + @Override + public boolean contains(FieldSet o) { + if (o instanceof BodyFields || o instanceof DocIdOnly || o instanceof NoFields) { + return true; + } + + if (o instanceof Field) { + return !((Field) o).isHeader(); + } + + if (o instanceof FieldCollection) { + FieldCollection c = (FieldCollection)o; + for (Field f : c) { + if (f.isHeader()) { + return false; + } + } + + return true; + } else { + return false; + } + } + + @Override + public FieldSet clone() throws CloneNotSupportedException { + return new BodyFields(); + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/DocIdOnly.java b/document/src/main/java/com/yahoo/document/fieldset/DocIdOnly.java new file mode 100644 index 00000000000..96deedf34f0 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/DocIdOnly.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +/** + * Created with IntelliJ IDEA. + * User: thomasg + * Date: 4/25/12 + * Time: 3:21 PM + * To change this template use File | Settings | File Templates. + */ +public class DocIdOnly implements FieldSet { + @Override + public boolean contains(FieldSet o) { + return (o instanceof DocIdOnly || o instanceof NoFields); + } + + @Override + public FieldSet clone() throws CloneNotSupportedException { + return new DocIdOnly(); + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/FieldCollection.java b/document/src/main/java/com/yahoo/document/fieldset/FieldCollection.java new file mode 100644 index 00000000000..bec47bb9c2e --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/FieldCollection.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; + +import java.util.ArrayList; + +public class FieldCollection extends ArrayList<Field> implements FieldSet { + DocumentType docType; + + public FieldCollection(DocumentType type) { + docType = type; + } + + public DocumentType getDocumentType() { + return docType; + } + + @Override + public boolean contains(FieldSet o) { + if (o instanceof DocIdOnly || o instanceof NoFields) { + return true; + } + + if (o instanceof Field) { + return super.contains(o); + } else if (o instanceof FieldCollection) { + FieldCollection c = (FieldCollection)o; + + for (Field f : c) { + if (!super.contains(f)) { + return false; + } + } + + return true; + } else { + return false; + } + } + + @Override + public FieldSet clone() { + FieldCollection c = new FieldCollection(docType); + c.addAll(this); + return c; + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/FieldSet.java b/document/src/main/java/com/yahoo/document/fieldset/FieldSet.java new file mode 100644 index 00000000000..0d0b9244e35 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/FieldSet.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +import com.yahoo.document.Document; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; + +import java.lang.Object; +import java.util.Iterator; +import java.util.Map; + +/** + * TODO: Move to Java and implement. + */ +public interface FieldSet { + public boolean contains(FieldSet o); + + public FieldSet clone() throws CloneNotSupportedException; +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java b/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java new file mode 100644 index 00000000000..c0da7fd9bc7 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/FieldSetRepo.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; + +import java.lang.String; +import java.util.*; + +/** + * TODO: Move to document and implement + */ +public class FieldSetRepo { + + FieldSet parseSpecialValues(String name) + { + if (name.equals("[id]")) { return new DocIdOnly(); } + else if (name.equals("[all]")) { return (new AllFields()); } + else if (name.equals("[none]")) { return (new NoFields()); } + else if (name.equals("[header]")) { return (new HeaderFields()); } + else if (name.equals("[docid]")) { return (new DocIdOnly()); } + else if (name.equals("[body]")) { return (new BodyFields()); } + else { + throw new IllegalArgumentException( + "The only special names (enclosed in '[]') allowed are " + + "id, all, none, header, body"); + } + } + + FieldSet parseFieldCollection(DocumentTypeManager docMan, String docType, String fieldNames) { + DocumentType type = docMan.getDocumentType(docType); + if (type == null) { + throw new IllegalArgumentException("Unknown document type " + docType); + } + + StringTokenizer tokenizer = new StringTokenizer(fieldNames, ","); + FieldCollection collection = new FieldCollection(type); + + for (; tokenizer.hasMoreTokens(); ) { + String token = tokenizer.nextToken(); + Field f = type.getField(token); + if (f == null) { + throw new IllegalArgumentException("No such field " + token); + } + collection.add(f); + } + + return collection; + } + + public FieldSet parse(DocumentTypeManager docMan, String fieldSet) { + if (fieldSet.length() == 0) { + throw new IllegalArgumentException("Illegal field set value \"\""); + } + + if (fieldSet.startsWith("[")) { + return parseSpecialValues(fieldSet); + } + + StringTokenizer tokenizer = new StringTokenizer(fieldSet, ":"); + if (tokenizer.countTokens() != 2) { + throw new IllegalArgumentException( + "The field set list must consist of a document type, " + + "then a colon (:), then a comma-separated list of field names"); + } + + String type = tokenizer.nextToken(); + String fields = tokenizer.nextToken(); + + return parseFieldCollection(docMan, type, fields); + } + + public String serialize(FieldSet fieldSet) { + if (fieldSet instanceof Field) { + return ((Field)fieldSet).getName(); + } else if (fieldSet instanceof FieldCollection) { + FieldCollection c = ((FieldCollection)fieldSet); + + StringBuffer buffer = new StringBuffer(); + for (Field f : c) { + if (buffer.length() == 0) { + buffer.append(c.getDocumentType().getName()); + buffer.append(":"); + } else { + buffer.append(","); + } + buffer.append(f.getName()); + } + + return buffer.toString(); + } else if (fieldSet instanceof AllFields) { + return "[all]"; + } else if (fieldSet instanceof NoFields) { + return "[none]"; + } else if (fieldSet instanceof BodyFields) { + return "[body]"; + } else if (fieldSet instanceof HeaderFields) { + return "[header]"; + } else if (fieldSet instanceof DocIdOnly) { + return "[docid]"; + } else { + throw new IllegalArgumentException("Unknown field set type " + fieldSet); + } + } + + + /** + * Copies fields from one document to another based on whether the fields match the given + * fieldset. + */ + public void copyFields(Document source, Document target, FieldSet fieldSet) { + for (Iterator<Map.Entry<Field, FieldValue>> i = source.iterator(); i.hasNext();) { + Map.Entry<Field, FieldValue> v = i.next(); + + if (fieldSet.contains(v.getKey())) { + target.setFieldValue(v.getKey(), v.getValue()); + } + } + } + + /** + * Strips all fields not wanted by the given field set from the document. + */ + public void stripFields(Document target, FieldSet fieldSet) { + List<Field> toStrip = new ArrayList<Field>(); + for (Iterator<Map.Entry<Field, FieldValue>> i = target.iterator(); i.hasNext();) { + Map.Entry<Field, FieldValue> v = i.next(); + + if (!fieldSet.contains(v.getKey())) { + toStrip.add(v.getKey()); + } + } + + for (Field f : toStrip) { + target.removeFieldValue(f); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/HeaderFields.java b/document/src/main/java/com/yahoo/document/fieldset/HeaderFields.java new file mode 100644 index 00000000000..a9e9375f9ac --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/HeaderFields.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +import com.yahoo.document.Field; + +/** + * Created with IntelliJ IDEA. + * User: thomasg + * Date: 4/25/12 + * Time: 3:18 PM + * To change this template use File | Settings | File Templates. + */ +public class HeaderFields implements FieldSet { + @Override + public boolean contains(FieldSet o) { + if (o instanceof HeaderFields || o instanceof DocIdOnly || o instanceof NoFields) { + return true; + } + + if (o instanceof Field) { + return ((Field)o).isHeader(); + } + + if (o instanceof FieldCollection) { + FieldCollection c = (FieldCollection)o; + for (Field f : c) { + if (!f.isHeader()) { + return false; + } + } + + return true; + } else { + return false; + } + } + + @Override + public FieldSet clone() throws CloneNotSupportedException { + return new HeaderFields(); + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/NoFields.java b/document/src/main/java/com/yahoo/document/fieldset/NoFields.java new file mode 100644 index 00000000000..43d9412f94d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/NoFields.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.fieldset; + +/** + * Created with IntelliJ IDEA. + * User: thomasg + * Date: 4/25/12 + * Time: 3:18 PM + * To change this template use File | Settings | File Templates. + */ +public class NoFields implements FieldSet { + @Override + public boolean contains(FieldSet o) { + return (o instanceof NoFields); + } + + @Override + public FieldSet clone() throws CloneNotSupportedException { + return new NoFields(); + } +} diff --git a/document/src/main/java/com/yahoo/document/fieldset/package-info.java b/document/src/main/java/com/yahoo/document/fieldset/package-info.java new file mode 100644 index 00000000000..33534f3396c --- /dev/null +++ b/document/src/main/java/com/yahoo/document/fieldset/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.fieldset; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/idstring/DocIdString.java b/document/src/main/java/com/yahoo/document/idstring/DocIdString.java new file mode 100644 index 00000000000..85ed7451fbe --- /dev/null +++ b/document/src/main/java/com/yahoo/document/idstring/DocIdString.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.idstring; + +import com.yahoo.collections.MD5; +import com.yahoo.text.Utf8; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * Representation of doc scheme in document IDs. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocIdString extends IdString { + /** + * Create a doc scheme object. + * <code>doc:<namespace>:<namespaceSpecific></code> + * + * @param namespace The namespace of this document id. + * @param namespaceSpecific The namespace specific part. + */ + public DocIdString(String namespace, String namespaceSpecific) { + super(Scheme.doc, namespace, namespaceSpecific); + } + + /** + * Get the location of this document id. The location is used for distribution + * in clusters. For the doc scheme, the location is a hash of the whole id. + * + * @return The 64 bit location. + */ + public long getLocation() { + long result = 0; + byte[] md5sum = MD5.md5.get().digest(Utf8.toBytes(toString())); + for (int i=0; i<8; ++i) { + result |= (md5sum[i] & 0xFFl) << (8*i); + } + + return result; + } + + /** Get the scheme specific part. Which is non-existing for doc scheme. */ + public String getSchemeSpecific() { + return ""; + } +} diff --git a/document/src/main/java/com/yahoo/document/idstring/GroupDocIdString.java b/document/src/main/java/com/yahoo/document/idstring/GroupDocIdString.java new file mode 100644 index 00000000000..aed7c78d0ef --- /dev/null +++ b/document/src/main/java/com/yahoo/document/idstring/GroupDocIdString.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.idstring; + +import com.yahoo.collections.MD5; +import com.yahoo.text.Utf8; + +import java.security.MessageDigest; + +/** + * Representation of groupdoc scheme in document IDs. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class GroupDocIdString extends IdString { + String group; + + /** + * Create a groupdoc scheme object. + * <code>groupdoc:<namespace>:<group>:<namespaceSpecific></code> + * + * @param namespace The namespace of this document id. + * @param group The groupname of this groupdoc id. + * @param namespaceSpecific The namespace specific part. + */ + public GroupDocIdString(String namespace, String group, String namespaceSpecific) { + super(Scheme.groupdoc, namespace, namespaceSpecific); + this.group = group; + } + + /** + * Get the location of this document id. The location is used for distribution + * in clusters. For the groupdoc scheme, the location is a hash of the groupname. + * + * @return The 64 bit location. + */ + public long getLocation() { + long result = 0; + try{ + byte[] md5sum = MD5.md5.get().digest(Utf8.toBytes(group)); + for (int i=0; i<8; ++i) { + result |= (md5sum[i] & 0xFFl) << (8*i); + } + } catch (Exception e) { + e.printStackTrace(); // TODO: FIXME! + } + return result; + } + + /** Get the scheme specific part. Which is for a groupdoc, is the groupdoc and a colon. */ + public String getSchemeSpecific() { + return group + ":"; + } + + @Override + public boolean hasGroup() { + return true; + } + + /** @return Get the groupname of this id. */ + @Override + public String getGroup() { + return group; + } +} diff --git a/document/src/main/java/com/yahoo/document/idstring/IdIdString.java b/document/src/main/java/com/yahoo/document/idstring/IdIdString.java new file mode 100644 index 00000000000..6fd5c578ee8 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/idstring/IdIdString.java @@ -0,0 +1,132 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.idstring; + +import com.yahoo.collections.MD5; +import com.yahoo.text.Utf8; + +/** + * Created with IntelliJ IDEA. + * User: magnarn + * Date: 10/15/12 + * Time: 11:02 AM + */ +public class IdIdString extends IdString { + private String type; + private String group; + private long location; + private boolean hasGroup; + private boolean hasNumber; + + public static String replaceType(String id, String typeName) { + int typeStartPos = id.indexOf(":", 3) + 1; + int typeEndPos = id.indexOf(":", typeStartPos); + return id.substring(0, typeStartPos) + typeName + id.substring(typeEndPos); + } + + + private static long makeLocation(String s) { + long result = 0; + byte[] md5sum = MD5.md5.get().digest(Utf8.toBytes(s)); + for (int i=0; i<8; ++i) { + result |= (md5sum[i] & 0xFFl) << (8*i); + } + + return result; + } + + /** + * Create an id scheme object. + * <code>doc:<namespace>:<documentType>:<key-value-pairs>:<namespaceSpecific></code> + * + * @param namespace The namespace of this document id. + * @param type The type of this document id. + * @param keyValues The key/value pairs of this document id. + * @param localId The namespace specific part. + */ + public IdIdString(String namespace, String type, String keyValues, String localId) { + super(Scheme.id, namespace, localId); + this.type = type; + boolean hasSetLocation = false; + for(String pair : keyValues.split(",")) { + int pos = pair.indexOf('='); + if (pos == -1) { + if (pair.equals("")) { // empty pair is ok + continue; + } + throw new IllegalArgumentException("Illegal key-value pair '" + pair + "'"); + } + String key = pair.substring(0, pos); + String value = pair.substring(pos + 1); + switch(key) { + case "n": + if (hasSetLocation) { + throw new IllegalArgumentException("Illegal key combination in " + keyValues); + } + location = Long.parseLong(value); + hasSetLocation = true; + hasNumber = true; + break; + case "g": + if (hasSetLocation) { + throw new IllegalArgumentException("Illegal key combination in " + keyValues); + } + location = makeLocation(value); + hasSetLocation = true; + hasGroup = true; + group = value; + break; + default: + throw new IllegalArgumentException("Illegal key '" + key + "'"); + } + } + if (!hasSetLocation) { + location = makeLocation(localId); + } + } + + @Override + public long getLocation() { + return location; + } + + @Override + public String getSchemeSpecific() { + if (hasGroup) { + return type + ":g=" + group + ":"; + } else if (hasNumber) { + return type + ":n=" + location + ":"; + } else { + return type + "::"; + } + } + + @Override + public boolean hasDocType() { + return true; + } + + @Override + public String getDocType() { + return type; + } + + @Override + public boolean hasGroup() { + return hasGroup; + } + + @Override + public String getGroup() { + return group; + } + + @Override + public boolean hasNumber() { + return hasNumber; + } + + @Override + public long getNumber() { + return location; + } +} diff --git a/document/src/main/java/com/yahoo/document/idstring/IdString.java b/document/src/main/java/com/yahoo/document/idstring/IdString.java new file mode 100644 index 00000000000..55fd601dd0d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/idstring/IdString.java @@ -0,0 +1,219 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.idstring; + +import com.yahoo.text.Utf8String; + +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * To be used with DocumentId constructor. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public abstract class IdString { + + public boolean hasDocType() { + return false; + } + + public String getDocType() { + return ""; + } + + public boolean hasGroup() { + return false; + } + + public boolean hasNumber() { + return false; + } + + public long getNumber() { + return 0; + } + + public String getGroup() { + return ""; + } + + public class GidModifier { + public int usedBits; + public long value; + } + + public enum Scheme { doc, userdoc, groupdoc, orderdoc, id } + final Scheme scheme; + final String namespace; + final String namespaceSpecific; + Utf8String cache; + + public static int[] generateOrderDocParams(String scheme) { + int parenPos = scheme.indexOf("("); + int endParenPos = scheme.indexOf(")"); + + if (parenPos == -1 || endParenPos == -1) { + throw new IllegalArgumentException("Unparseable scheme " + scheme + ": Must be on the form orderdoc(width, division)"); + } + + String params = scheme.substring(parenPos + 1, endParenPos); + String[] vals = params.split(","); + + if (vals.length != 2) { + throw new IllegalArgumentException("Unparseable scheme " + scheme + ": Must be on the form orderdoc(width, division)"); + } + + int[] retVal = new int[2]; + + try { + retVal[0] = Integer.parseInt(vals[0]); + retVal[1] = Integer.parseInt(vals[1]); + return retVal; + } catch (Exception e) { + throw new IllegalArgumentException("Unparseable scheme " + scheme + ": Must be on the form orderdoc(width, division)"); + } + } + + public static IdString createIdString(String id) { + String namespace; + long userId; + String group; + long ordering; + + int schemePos = id.indexOf(":"); + if (schemePos < 0) { + throw new IllegalArgumentException("Unparseable id '" + id + "': Scheme missing"); + } + + //Find scheme + String schemeStr = id.substring(0, schemePos); + int currPos = schemePos + 1; + + //Find namespace + int colonPos = id.indexOf(":", currPos); + if (colonPos < 0) { + throw new IllegalArgumentException("Unparseable id '" + id + "': Namespace missing"); + } else { + namespace = id.substring(currPos, colonPos); + + if (namespace.length() == 0) { + throw new IllegalArgumentException("Unparseable id '" + id + "': Namespace must be non-empty"); + } + + currPos = colonPos + 1; + } + + if (schemeStr.equals("id")) { + colonPos = id.indexOf(":", currPos); + if (colonPos < 0) { + throw new IllegalArgumentException("Unparseable id '" + id + "': Document type missing"); + } + String type = id.substring(currPos, colonPos); + currPos = colonPos + 1; + colonPos = id.indexOf(":", currPos); + if (colonPos < 0) { + throw new IllegalArgumentException("Unparseable id '" + id + "': Key/value section missing"); + } + String keyValues = id.substring(currPos, colonPos); + + currPos = colonPos + 1; + return new IdIdString(namespace, type, keyValues, id.substring(currPos)); + + } if (schemeStr.equals("doc")) { + return new DocIdString(namespace, id.substring(currPos)); + } else if (schemeStr.equals("userdoc")) { + colonPos = id.indexOf(":", currPos); + if (colonPos < 0) { + throw new IllegalArgumentException("Unparseable id '" + id + "': User id missing"); + } + + try { + userId = new BigInteger(id.substring(currPos, colonPos)).longValue(); + } catch (IllegalArgumentException iae) { + throw new IllegalArgumentException("Unparseable id '" + id + "': " + iae.getMessage(), iae.getCause()); + } + + currPos = colonPos + 1; + return new UserDocIdString(namespace, userId, id.substring(currPos)); + } else if (schemeStr.equals("groupdoc")) { + colonPos = id.indexOf(":", currPos); + + if (colonPos < 0) { + throw new IllegalArgumentException("Unparseable id '" + id + "': Group id missing"); + } + + group = id.substring(currPos, colonPos); + currPos = colonPos + 1; + return new GroupDocIdString(namespace, group, id.substring(currPos)); + } else if (schemeStr.indexOf("orderdoc") == 0) { + int[] params = generateOrderDocParams(schemeStr); + + colonPos = id.indexOf(":", currPos); + + if (colonPos < 0) { + throw new IllegalArgumentException("Unparseable id '" + id + "': Group id missing"); + } + + group = id.substring(currPos, colonPos); + + currPos = colonPos + 1; + + colonPos = id.indexOf(":", currPos); + if (colonPos < 0) { + throw new IllegalArgumentException("Unparseable id '" + id + "': Ordering missing"); + } + + try { + ordering = Long.parseLong(id.substring(currPos, colonPos)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unparseable id '" + id + "': " + e.getMessage(), e.getCause()); + } + + currPos = colonPos + 1; + return new OrderDocIdString(namespace, group, params[0], params[1], ordering, id.substring(currPos)); + } else { + throw new IllegalArgumentException("Unknown id scheme '" + schemeStr + "'"); + } + } + + protected IdString(Scheme scheme, String namespace, String namespaceSpecific) { + this.scheme = scheme; + this.namespace = namespace; + this.namespaceSpecific = namespaceSpecific; + } + + public Scheme getType() { return scheme; } + + public String getNamespace() { return namespace; } + public String getNamespaceSpecific() { return namespaceSpecific; } + public abstract long getLocation(); + public String getSchemeParameters() { return ""; } + public abstract String getSchemeSpecific(); + public GidModifier getGidModifier() { return null; } + + public boolean equals(Object o) { + return (o instanceof IdString && o.toString().equals(toString())); + } + + public int hashCode() { + return toString().hashCode(); + } + + private Utf8String createToString() { + return new Utf8String(scheme.toString() + getSchemeParameters() + ':' + namespace + ':' + getSchemeSpecific() + namespaceSpecific); + } + public String toString() { + if (cache == null) { + cache = createToString(); + } + return cache.toString(); + } + public Utf8String toUtf8() { + if (cache == null) { + cache = createToString(); + } + return cache; + } + +} diff --git a/document/src/main/java/com/yahoo/document/idstring/OrderDocIdString.java b/document/src/main/java/com/yahoo/document/idstring/OrderDocIdString.java new file mode 100644 index 00000000000..111be0110b5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/idstring/OrderDocIdString.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.idstring; + +import com.yahoo.collections.MD5; +import com.yahoo.text.Utf8; + +import java.security.MessageDigest; + +/** + * Representation of groupdoc scheme in document IDs. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class OrderDocIdString extends IdString { + String group; + int widthBits; + int divisionBits; + long ordering; + long location; + + /** + * Create a groupdoc scheme object. + * <code>groupdoc:<namespace>:<group>:<namespaceSpecific></code> + * + * @param namespace The namespace of this document id. + * @param group The groupname of this groupdoc id. + * @param widthBits The number of bits used for the width of the data set + * @param divisionBits The number of bits used for the smalles partitioning of the data set + * @param ordering A value used to order documents of this type. + * @param namespaceSpecific The namespace specific part. + */ + public OrderDocIdString(String namespace, String group, int widthBits, int divisionBits, long ordering, String namespaceSpecific) { + super(Scheme.orderdoc, namespace, namespaceSpecific); + this.group = group; + this.widthBits = widthBits; + this.divisionBits = divisionBits; + this.ordering = ordering; + + try { + this.location = Long.parseLong(group); + } catch (Exception foo) { + location = 0; + byte[] md5sum = MD5.md5.get().digest(Utf8.toBytes(group)); + for (int i=0; i<8; ++i) { + location |= (md5sum[i] & 0xFFl) << (8*i); + } + } + } + + /** + * Get the location of this document id. The location is used for distribution + * in clusters. For the orderdoc scheme, the location is a hash of the groupname or just the number specified. + * + * @return The 64 bit location. + */ + public long getLocation() { + return location; + } + + public String getSchemeParameters() { + return "(" + widthBits + "," + divisionBits + ")"; + } + + /** Get the scheme specific part. */ + public String getSchemeSpecific() { + return group + ":" + ordering + ":"; + } + + public GidModifier getGidModifier() { + GidModifier gm = new GidModifier(); + gm.usedBits = widthBits - divisionBits; + long gidBits = (ordering << (64 - widthBits)); + gidBits = Long.reverse(gidBits); + long gidMask = (0xFFFFFFFFFFFFFFFFl >>> (64 - gm.usedBits)); + gidBits &= gidMask; + gm.value = gidBits; + return gm; + } + + @Override + public boolean hasGroup() { + return true; + } + + /** @return Get the groupname of this id. */ + @Override + public String getGroup() { + return group; + } + + @Override + public boolean hasNumber() { + return true; + } + + @Override + public long getNumber() { + return location; + } + + public long getUserId() { + return location; + } + + public int getWidthBits() { + return widthBits; + } + + public int getDivisionBits() { + return divisionBits; + } + + public long getOrdering() { + return ordering; + } +} diff --git a/document/src/main/java/com/yahoo/document/idstring/UserDocIdString.java b/document/src/main/java/com/yahoo/document/idstring/UserDocIdString.java new file mode 100644 index 00000000000..631c199e6bf --- /dev/null +++ b/document/src/main/java/com/yahoo/document/idstring/UserDocIdString.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.idstring; + +import java.math.BigInteger; + +/** + * Representation of userdoc scheme in document IDs. A user id is any 64 bit + * number. Note that internally, these are handled as unsigned values. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class UserDocIdString extends IdString { + long userId; + + /** + * Create a userdoc scheme object. + * <code>userdoc:<namespace>:<userid>:<namespaceSpecific></code> + * + * @param namespace The namespace of this document id. + * @param userId 64 bit user id of this userdoc id. + * @param namespaceSpecific The namespace specific part. + */ + public UserDocIdString(String namespace, long userId, String namespaceSpecific) { + super(Scheme.userdoc, namespace, namespaceSpecific); + this.userId = userId; + } + + @Override + public boolean hasNumber() { + return true; + } + + @Override + public long getNumber() { + return userId; + } + + /** + * Get the location of this document id. The location is used for distribution + * in clusters. For the userdoc scheme, the location equals the user id. + * + * @return The 64 bit location. + */ + public long getLocation() { return userId; } + + /** Get the scheme specific part. Which for a userdoc, is the userid and a colon. */ + public String getSchemeSpecific() { + BigInteger uid = BigInteger.ZERO; + for (int i=0; i<64; i++) { + if ((userId >>> i & 0x1) == 1) { + uid = uid.setBit(i); + } + } + return uid.toString() + ":"; + } + + /** @return Get the user id of this id. */ + public long getUserId() { return userId; } +} diff --git a/document/src/main/java/com/yahoo/document/idstring/package-info.java b/document/src/main/java/com/yahoo/document/idstring/package-info.java new file mode 100644 index 00000000000..4ddd539ccd5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/idstring/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.idstring; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java new file mode 100644 index 00000000000..5d329554192 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import java.io.InputStream; + +import com.fasterxml.jackson.core.JsonFactory; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.vespaxmlparser.FeedReader; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; + + +/** + * Facade between JsonReader and the FeedReader API. + * + * <p> + * The feed reader will take ownership of the input stream and close it when the + * last parseable document has been read. + * + * @author steinar + */ +public class JsonFeedReader implements FeedReader { + private final JsonReader reader; + private InputStream stream; + private static final JsonFactory jsonFactory = new JsonFactory(); + + public JsonFeedReader(InputStream stream, DocumentTypeManager docMan) { + reader = new JsonReader(docMan, stream, jsonFactory); + this.stream = stream; + } + + @Override + public void read(Operation operation) throws Exception { + DocumentOperation documentOperation = reader.next(); + + if (documentOperation == null) { + stream.close(); + operation.setInvalid(); + return; + } + + if (documentOperation instanceof DocumentUpdate) { + operation.setDocumentUpdate((DocumentUpdate) documentOperation); + } else if (documentOperation instanceof DocumentRemove) { + operation.setRemove(documentOperation.getId()); + } else if (documentOperation instanceof DocumentPut) { + operation.setDocument(((DocumentPut) documentOperation).getDocument()); + } else { + throw new IllegalStateException("Got unknown class from JSON reader: " + documentOperation.getClass().getName()); + } + + operation.setCondition(documentOperation.getCondition()); + } + +} diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java new file mode 100644 index 00000000000..e5402d617bd --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -0,0 +1,773 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.annotations.Beta; +import com.google.common.base.Preconditions; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.TokenBuffer.Token; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.tensor.MapTensorBuilder; +import org.apache.commons.codec.binary.Base64; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Initialize Vespa documents/updates/removes from an InputStream containing a + * valid JSON representation of a feed. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @since 5.1.25 + */ +@Beta +public class JsonReader { + + private enum FieldOperation { + ADD, REMOVE + } + + static final String MAP_KEY = "key"; + static final String MAP_VALUE = "value"; + static final String FIELDS = "fields"; + static final String REMOVE = "remove"; + static final String UPDATE_INCREMENT = "increment"; + static final String UPDATE_DECREMENT = "decrement"; + static final String UPDATE_MULTIPLY = "multiply"; + static final String UPDATE_DIVIDE = "divide"; + static final String TENSOR_DIMENSIONS = "dimensions"; + static final String TENSOR_CELLS = "cells"; + static final String TENSOR_ADDRESS = "address"; + static final String TENSOR_VALUE = "value"; + + private static final String UPDATE = "update"; + private static final String PUT = "put"; + private static final String ID = "id"; + private static final String CONDITION = "condition"; + private static final String CREATE_IF_NON_EXISTENT = "create"; + private static final String UPDATE_ASSIGN = "assign"; + private static final String UPDATE_REMOVE = "remove"; + private static final String UPDATE_MATCH = "match"; + private static final String UPDATE_ADD = "add"; + private static final String UPDATE_ELEMENT = "element"; + + private final JsonParser parser; + private TokenBuffer buffer = new TokenBuffer(); + private final DocumentTypeManager typeManager; + private ReaderState state = ReaderState.AT_START; + + static class DocumentParseInfo { + public DocumentId documentId; + public Optional<Boolean> create = Optional.empty(); + Optional<String> condition = Optional.empty(); + SupportedOperation operationType = null; + } + + enum SupportedOperation { + PUT, UPDATE, REMOVE + } + + enum ReaderState { + AT_START, READING, END_OF_FEED + } + + public JsonReader(DocumentTypeManager typeManager, InputStream input, JsonFactory parserFactory) { + this.typeManager = typeManager; + + try { + parser = parserFactory.createParser(input); + } catch (IOException e) { + state = ReaderState.END_OF_FEED; + throw new RuntimeException(e); + } + } + + /** + * Reads a single operation. The operation is not expected to be part of an array. It only reads FIELDS. + * @param operationType the type of operation (update or put) + * @param docIdString document ID. + * @return the document + */ + public DocumentOperation readSingleDocument(SupportedOperation operationType, String docIdString) { + DocumentId docId = new DocumentId(docIdString); + DocumentParseInfo documentParseInfo = parseToDocumentsFieldsAndInsertFieldsIntoBuffer(docId); + documentParseInfo.operationType = operationType; + DocumentOperation operation = createDocumentOperation(documentParseInfo); + operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition)); + return operation; + } + + public DocumentOperation next() { + switch (state) { + case AT_START: + JsonToken t = nextToken(); + expectArrayStart(t); + state = ReaderState.READING; + break; + case END_OF_FEED: + return null; + case READING: + break; + } + + Optional<DocumentParseInfo> documentParseInfo = parseDocument(); + + if (! documentParseInfo.isPresent()) { + state = ReaderState.END_OF_FEED; + return null; + } + DocumentOperation operation = createDocumentOperation(documentParseInfo.get()); + operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.get().condition)); + return operation; + } + + private DocumentOperation createDocumentOperation(DocumentParseInfo documentParseInfo) { + DocumentType documentType = getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager); + final DocumentOperation documentOperation; + try { + switch (documentParseInfo.operationType) { + case PUT: + documentOperation = new DocumentPut(new Document(documentType, documentParseInfo.documentId)); + readPut((DocumentPut) documentOperation); + verifyEndState(); + break; + case REMOVE: + documentOperation = new DocumentRemove(documentParseInfo.documentId); + break; + case UPDATE: + documentOperation = new DocumentUpdate(documentType, documentParseInfo.documentId); + readUpdate((DocumentUpdate) documentOperation); + verifyEndState(); + break; + default: + throw new IllegalStateException("Implementation out of sync with itself. This is a bug."); + } + } catch (JsonReaderException e) { + throw JsonReaderException.addDocId(e, documentParseInfo.documentId); + } + if (documentParseInfo.create.isPresent()) { + if (!(documentOperation instanceof DocumentUpdate)) { + throw new RuntimeException("Could not set create flag on non update operation."); + } + DocumentUpdate update = (DocumentUpdate) documentOperation; + update.setCreateIfNonExistent(documentParseInfo.create.get()); + } + return documentOperation; + } + + void readUpdate(DocumentUpdate next) { + if (buffer.size() == 0) { + bufferFields(nextToken()); + } + populateUpdateFromBuffer(next); + } + + void readPut(DocumentPut put) { + if (buffer.size() == 0) { + bufferFields(nextToken()); + } + JsonToken t = buffer.currentToken(); + try { + populateComposite(put.getDocument(), t); + } catch (JsonReaderException e) { + throw JsonReaderException.addDocId(e, put.getId()); + } + } + + private DocumentParseInfo parseToDocumentsFieldsAndInsertFieldsIntoBuffer(DocumentId documentId) { + long indentLevel = 0; + DocumentParseInfo documentParseInfo = new DocumentParseInfo(); + documentParseInfo.documentId = documentId; + while (true) { + // we should now be at the start of a feed operation or at the end of the feed + JsonToken t = nextToken(); + if (t == null) { + throw new IllegalArgumentException("Could not read document, no document?"); + } + switch (t) { + case START_OBJECT: + indentLevel++; + break; + case END_OBJECT: + indentLevel--; + break; + case START_ARRAY: + indentLevel+=10000L; + break; + case END_ARRAY: + indentLevel-=10000L; + break; + } + if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) { + try { + if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue()); + continue; + } + } catch (IOException e) { + throw new RuntimeException("Got IO exception while parsing document", e); + } + } + if (indentLevel == 2L && t == JsonToken.START_OBJECT) { + + try { + if (!FIELDS.equals(parser.getCurrentName())) { + continue; + } + } catch (IOException e) { + throw new RuntimeException("Got IO exception while parsing document", e); + } + bufferFields(t); + break; + } + } + return documentParseInfo; + } + + private void verifyEndState() { + Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation"); + expectObjectEnd(buffer.currentToken()); + Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation"); + Preconditions.checkState(buffer.size() == 0, "Dangling data at end of operation"); + } + + private void populateUpdateFromBuffer(DocumentUpdate update) { + expectObjectStart(buffer.currentToken()); + int localNesting = buffer.nesting(); + JsonToken t = buffer.next(); + + while (localNesting <= buffer.nesting()) { + expectObjectStart(t); + String fieldName = buffer.currentName(); + Field field = update.getType().getField(fieldName); + addFieldUpdates(update, field); + t = buffer.next(); + } + } + + private void addFieldUpdates(DocumentUpdate update, Field field) { + validateFieldUpdates(update, field); + int localNesting = buffer.nesting(); + FieldUpdate fieldUpdate = FieldUpdate.create(field); + + buffer.next(); + while (localNesting <= buffer.nesting()) { + switch (buffer.currentName()) { + case UPDATE_REMOVE: + createAddsOrRemoves(field, fieldUpdate, FieldOperation.REMOVE); + break; + case UPDATE_ADD: + createAddsOrRemoves(field, fieldUpdate, FieldOperation.ADD); + break; + case UPDATE_MATCH: + fieldUpdate.addValueUpdate(createMapUpdate(field)); + break; + default: + String action = buffer.currentName(); + fieldUpdate.addValueUpdate(readSingleUpdate(field.getDataType(), action)); + } + buffer.next(); + } + update.addFieldUpdate(fieldUpdate); + } + + private static void validateFieldUpdates(DocumentUpdate update, Field field) { + if (field.getDataType() == DataType.TENSOR) { + throw new IllegalArgumentException("Updates to fields of type TENSOR is not yet supported (" + + "id='" + update.getId().toString() + "', field='" + field.getName() + "')"); + } + } + + @SuppressWarnings("rawtypes") + private ValueUpdate createMapUpdate(Field field) { + buffer.next(); + MapValueUpdate m = (MapValueUpdate) createMapUpdate(field.getDataType(), null, null); + buffer.next(); + // must generate the field value in parallell with the actual + return m; + + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private ValueUpdate createMapUpdate(DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) { + TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT); + if (UPDATE_ELEMENT.equals(buffer.currentName())) { + buffer.next(); + } + + FieldValue key = keyTypeForMapUpdate(element, currentLevel); + if (keyParent != null) { + ((CollectionFieldValue) keyParent).add(key); + } + // structure is: [(match + element)*, (element + action)] + // match will always have element, and either match or action + if (!UPDATE_MATCH.equals(buffer.currentName())) { + // we have reached an action... + if (topLevelKey == null) { + return ValueUpdate.createMap(key, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName())); + } else { + return ValueUpdate.createMap(topLevelKey, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName())); + } + } else { + // next level of matching + if (topLevelKey == null) { + return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, key); + } else { + return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, topLevelKey); + } + } + } + + private DataType valueTypeForMapUpdate(DataType parentType) { + if (parentType instanceof WeightedSetDataType) { + return DataType.INT; + } else if (parentType instanceof CollectionDataType) { + return ((CollectionDataType) parentType).getNestedType(); + } else if (parentType instanceof MapDataType) { + return ((MapDataType) parentType).getValueType(); + } else { + throw new UnsupportedOperationException("Unexpected parent type: " + parentType); + } + } + + private FieldValue keyTypeForMapUpdate(Token element, DataType expectedType) { + FieldValue v; + if (expectedType instanceof ArrayDataType) { + v = new IntegerFieldValue(Integer.valueOf(element.text)); + } else if (expectedType instanceof WeightedSetDataType) { + v = ((WeightedSetDataType) expectedType).getNestedType().createFieldValue(element.text); + } else if (expectedType instanceof MapDataType) { + v = ((MapDataType) expectedType).getKeyType().createFieldValue(element.text); + } else { + throw new IllegalArgumentException("Container type " + expectedType + " not supported for match update."); + } + return v; + } + + @SuppressWarnings("rawtypes") + private ValueUpdate readSingleUpdate(DataType expectedType, String action) { + ValueUpdate update; + + switch (action) { + case UPDATE_ASSIGN: + update = (buffer.currentToken() == JsonToken.VALUE_NULL) + ? ValueUpdate.createClear() + : ValueUpdate.createAssign(readSingleValue(buffer.currentToken(), expectedType)); + break; + // double is silly, but it's what is used internally anyway + case UPDATE_INCREMENT: + update = ValueUpdate.createIncrement(Double.valueOf(buffer.currentText())); + break; + case UPDATE_DECREMENT: + update = ValueUpdate.createDecrement(Double.valueOf(buffer.currentText())); + break; + case UPDATE_MULTIPLY: + update = ValueUpdate.createMultiply(Double.valueOf(buffer.currentText())); + break; + case UPDATE_DIVIDE: + update = ValueUpdate.createDivide(Double.valueOf(buffer.currentText())); + break; + default: + throw new IllegalArgumentException("Operation \"" + buffer.currentName() + "\" not implemented."); + } + return update; + } + + // yes, this suppresswarnings ugliness is by intention, the code relies on + // the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) + private void createAddsOrRemoves(Field field, FieldUpdate update, FieldOperation op) { + FieldValue container = field.getDataType().createFieldValue(); + FieldUpdate singleUpdate; + int initNesting = buffer.nesting(); + JsonToken token; + + Preconditions.checkState(buffer.currentToken().isStructStart(), "Expected start of composite, got %s", buffer.currentToken()); + if (container instanceof CollectionFieldValue) { + token = buffer.next(); + DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType(); + if (container instanceof WeightedSet) { + // these are objects with string keys (which are the nested + // types) and values which are the weight + WeightedSet weightedSet = (WeightedSet) container; + fillWeightedSetUpdate(initNesting, valueType, weightedSet); + if (op == FieldOperation.REMOVE) { + singleUpdate = FieldUpdate.createRemoveAll(field, weightedSet); + } else { + singleUpdate = FieldUpdate.createAddAll(field, weightedSet); + + } + } else { + List<FieldValue> arrayContents = new ArrayList<>(); + token = fillArrayUpdate(initNesting, token, valueType, arrayContents); + if (token != JsonToken.END_ARRAY) { + throw new IllegalStateException("Expected END_ARRAY. Got '" + token + "'."); + } + if (op == FieldOperation.REMOVE) { + singleUpdate = FieldUpdate.createRemoveAll(field, arrayContents); + } else { + singleUpdate = FieldUpdate.createAddAll(field, arrayContents); + } + } + } else { + throw new UnsupportedOperationException( + "Trying to add or remove from a field of a type the reader does not know how to handle: " + + container.getClass().getName()); + } + expectCompositeEnd(buffer.currentToken()); + update.addAll(singleUpdate); + } + + private JsonToken fillArrayUpdate(int initNesting, JsonToken initToken, DataType valueType, List<FieldValue> arrayContents) { + JsonToken token = initToken; + while (buffer.nesting() >= initNesting) { + arrayContents.add(readSingleValue(token, valueType)); + token = buffer.next(); + } + return token; + } + + private void fillWeightedSetUpdate(int initNesting, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { + iterateThroughWeightedSet(initNesting, valueType, weightedSet); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private void iterateThroughWeightedSet(int initNesting, DataType valueType, WeightedSet weightedSet) { + while (buffer.nesting() >= initNesting) { + // XXX the keys are defined in the spec to always be represented as strings + FieldValue v = valueType.createFieldValue(buffer.currentName()); + weightedSet.put(v, Integer.valueOf(buffer.currentText())); + buffer.next(); + } + } + + // TODO populateComposite is extremely similar to add/remove, refactor + // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes" }) + private void populateComposite(FieldValue parent, JsonToken token) { + if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) { + throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'."); + } + if (parent instanceof CollectionFieldValue) { + DataType valueType = ((CollectionFieldValue) parent).getDataType().getNestedType(); + if (parent instanceof WeightedSet) { + fillWeightedSet(valueType, (WeightedSet) parent); + } else { + fillArray((CollectionFieldValue) parent, valueType); + } + } else if (parent instanceof MapFieldValue) { + fillMap((MapFieldValue) parent); + } else if (parent instanceof StructuredFieldValue) { + fillStruct((StructuredFieldValue) parent); + } else if (parent instanceof TensorFieldValue) { + fillTensor((TensorFieldValue) parent); + } else { + throw new IllegalStateException("Has created a composite field" + + " value the reader does not know how to handle: " + + parent.getClass().getName() + " This is a bug. token = " + token); + } + expectCompositeEnd(buffer.currentToken()); + } + + private void expectCompositeEnd(JsonToken token) { + Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token); + } + + private void fillStruct(StructuredFieldValue parent) { + // do note the order of initializing initNesting and token is relevant for empty docs + int initNesting = buffer.nesting(); + JsonToken token = buffer.next(); + + while (buffer.nesting() >= initNesting) { + Field f = getField(parent); + try { + FieldValue v = readSingleValue(token, f.getDataType()); + parent.setFieldValue(f, v); + token = buffer.next(); + } catch (IllegalArgumentException e) { + throw new JsonReaderException(f, e); + } + } + } + + private Field getField(StructuredFieldValue parent) { + Field f = parent.getField(buffer.currentName()); + if (f == null) { + throw new NullPointerException("Could not get field \"" + buffer.currentName() + + "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\"."); + } + return f; + } + + @SuppressWarnings({ "rawtypes", "cast", "unchecked" }) + private void fillMap(MapFieldValue parent) { + JsonToken token = buffer.currentToken(); + int initNesting = buffer.nesting(); + expectArrayStart(token); + token = buffer.next(); + DataType keyType = parent.getDataType().getKeyType(); + DataType valueType = parent.getDataType().getValueType(); + while (buffer.nesting() >= initNesting) { + FieldValue key = null; + FieldValue value = null; + expectObjectStart(token); + token = buffer.next(); + for (int i = 0; i < 2; ++i) { + if (MAP_KEY.equals(buffer.currentName())) { + key = readSingleValue(token, keyType); + } else if (MAP_VALUE.equals(buffer.currentName())) { + value = readSingleValue(token, valueType); + } + token = buffer.next(); + } + Preconditions.checkState(key != null && value != null, "Missing key or value for map entry."); + parent.put(key, value); + + expectObjectEnd(token); + token = buffer.next(); // array end or next entry + } + } + + private void expectArrayStart(JsonToken token) { + Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token); + } + + private void expectObjectStart(JsonToken token) { + Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token); + } + + private void expectObjectEnd(JsonToken token) { + Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token); + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private void fillArray(CollectionFieldValue parent, DataType valueType) { + int initNesting = buffer.nesting(); + expectArrayStart(buffer.currentToken()); + JsonToken token = buffer.next(); + while (buffer.nesting() >= initNesting) { + parent.add(readSingleValue(token, valueType)); + token = buffer.next(); + } + } + + private void fillWeightedSet(DataType valueType, + @SuppressWarnings("rawtypes") WeightedSet weightedSet) { + int initNesting = buffer.nesting(); + expectObjectStart(buffer.currentToken()); + buffer.next(); + iterateThroughWeightedSet(initNesting, valueType, weightedSet); + } + + private void fillTensor(TensorFieldValue tensorFieldValue) { + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + MapTensorBuilder tensorBuilder = new MapTensorBuilder(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + if (TENSOR_DIMENSIONS.equals(buffer.currentName())) { + readTensorDimensions(tensorBuilder); + } else if (TENSOR_CELLS.equals(buffer.currentName())) { + readTensorCells(tensorBuilder); + } + } + expectObjectEnd(buffer.currentToken()); + tensorFieldValue.assign(tensorBuilder.build()); + } + + private void readTensorDimensions(MapTensorBuilder tensorBuilder) { + expectArrayStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + if (buffer.currentToken().isScalarValue()) { + String dimension = buffer.currentText(); + tensorBuilder.dimension(dimension); + } + } + expectCompositeEnd(buffer.currentToken()); + } + + private void readTensorCells(MapTensorBuilder tensorBuilder) { + expectArrayStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + readTensorCell(tensorBuilder.cell()); + } + expectCompositeEnd(buffer.currentToken()); + } + + private void readTensorCell(MapTensorBuilder.CellBuilder cellBuilder) { + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + double cellValue = 0.0; + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + String currentName = buffer.currentName(); + if (TENSOR_ADDRESS.equals(currentName)) { + readTensorAddress(cellBuilder); + } else if (TENSOR_VALUE.equals(currentName)) { + cellValue = Double.valueOf(buffer.currentText()); + } + } + expectObjectEnd(buffer.currentToken()); + cellBuilder.value(cellValue); + } + + private void readTensorAddress(MapTensorBuilder.CellBuilder cellBuilder) { + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + String dimension = buffer.currentName(); + String label = buffer.currentText(); + cellBuilder.label(dimension, label); + } + expectObjectEnd(buffer.currentToken()); + } + + private FieldValue readSingleValue(JsonToken t, DataType expectedType) { + if (t.isScalarValue()) { + return readAtomic(expectedType); + } else { + FieldValue v = expectedType.createFieldValue(); + populateComposite(v, t); + return v; + } + } + + private FieldValue readAtomic(DataType expectedType) { + if (expectedType.equals(DataType.RAW)) { + return expectedType.createFieldValue(new Base64().decode(buffer.currentText())); + } else if (expectedType.equals(PositionDataType.INSTANCE)) { + return PositionDataType.fromString(buffer.currentText()); + } else { + return expectedType.createFieldValue(buffer.currentText()); + } + } + + private void bufferFields(JsonToken current) { + buffer.bufferObject(current, parser); + } + + private boolean jsonTokenIsBooleanOrString(JsonToken jsonToken) { + return jsonToken == JsonToken.VALUE_STRING || jsonToken == JsonToken.VALUE_TRUE || jsonToken == JsonToken.VALUE_FALSE; + } + + Optional<DocumentParseInfo> parseDocument() { + Optional<Boolean> create = Optional.empty(); + // we should now be at the start of a feed operation or at the end of the feed + JsonToken token = nextToken(); + if (token == JsonToken.END_ARRAY) { + return Optional.empty(); // end of feed + } + expectObjectStart(token); + + DocumentParseInfo documentParseInfo = new DocumentParseInfo(); + + while (true) { + try { + token = nextToken(); + if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) && + CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE); + continue; + } + if (token == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) { + documentParseInfo.condition = Optional.of(parser.getText()); + continue; + } + if (token == JsonToken.START_OBJECT) { + try { + if (!FIELDS.equals(parser.getCurrentName())) { + throw new IllegalArgumentException("Unexpected object key: " + parser.getCurrentName()); + } + } catch (IOException e) { + // TODO more specific wrapping + throw new RuntimeException(e); + } + bufferFields(token); + continue; + } + if (token == JsonToken.END_OBJECT) { + if (documentParseInfo.documentId == null) { + throw new RuntimeException("Did not find document operation"); + } + return Optional.of(documentParseInfo); + } + if (token == JsonToken.VALUE_STRING) { + documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName()); + documentParseInfo.documentId = new DocumentId(parser.getText()); + continue; + } + throw new RuntimeException("Expected document start or document operation."); + } catch (IOException e) { + throw new IllegalStateException(e); + } + + } + } + + private static SupportedOperation operationNameToOperationType(String operationName) { + switch (operationName) { + case PUT: + case ID: + return SupportedOperation.PUT; + case REMOVE: + return SupportedOperation.REMOVE; + case UPDATE: + return SupportedOperation.UPDATE; + default: + throw new IllegalArgumentException( + "Got " + operationName + " as document operation, only \"put\", " + + "\"remove\" and \"update\" are supported."); + } + } + + DocumentType readDocumentType(DocumentId docId) { + return getDocumentTypeFromString(docId.getDocType(), typeManager); + } + + private static DocumentType getDocumentTypeFromString(String docTypeString, DocumentTypeManager typeManager) { + final DocumentType docType = typeManager.getDocumentType(docTypeString); + if (docType == null) { + throw new IllegalArgumentException(String.format("Document type %s does not exist", docTypeString)); + } + return docType; + } + + private JsonToken nextToken() { + try { + return parser.nextValue(); + } catch (IOException e) { + // Jackson is not able to recover from structural parse errors + state = ReaderState.END_OF_FEED; + throw new RuntimeException(e); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/json/JsonReaderException.java b/document/src/main/java/com/yahoo/document/json/JsonReaderException.java new file mode 100644 index 00000000000..3346ecc3bd6 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/JsonReaderException.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.Field; + +/** + * @author bjorncs + */ +public class JsonReaderException extends RuntimeException { + public final DocumentId docId; + public final Field field; + public final Throwable cause; + + public JsonReaderException(DocumentId docId, Field field, Throwable cause) { + super(createErrorMessage(docId, field, cause), cause); + this.docId = docId; + this.field = field; + this.cause = cause; + } + + public JsonReaderException(Field field, Throwable cause) { + super(createErrorMessage(null, field, cause), cause); + this.docId = null; + this.field = field; + this.cause = cause; + } + + public static JsonReaderException addDocId(JsonReaderException oldException, DocumentId docId) { + return new JsonReaderException(docId, oldException.field, oldException.cause); + } + + private static String createErrorMessage(DocumentId docId, Field field, Throwable cause) { + return String.format("Error in document '%s' - could not parse field '%s' of type '%s': %s", + docId, field.getName(), field.getDataType().getName(), cause.getMessage()); + } + + public DocumentId getDocId() { + return docId; + } + + public Field getField() { + return field; + } +} diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java new file mode 100644 index 00000000000..79a1c040cbc --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -0,0 +1,473 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import com.yahoo.document.datatypes.*; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorAddress; +import org.apache.commons.codec.binary.Base64; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.serialization.DocumentWriter; +import com.yahoo.vespa.objects.FieldBase; +import com.yahoo.vespa.objects.Serializer; + +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * Serialize Document and other FieldValue instances as JSON. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class JsonWriter implements DocumentWriter { + + private static final JsonFactory jsonFactory = new JsonFactory(); + private final JsonGenerator generator; + private final Base64 base64Encoder = new Base64(); + + // I really hate exception unsafe constructors, but the alternative + // requires generator to not be a final + /** + * + * @param out + * the target output stream + * @throws RuntimeException + * if unable to create the internal JSON generator + */ + public JsonWriter(OutputStream out) { + this(createPrivateGenerator(out)); + } + + private static JsonGenerator createPrivateGenerator(OutputStream out) { + try { + return jsonFactory.createGenerator(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Create a Document writer which will write to the input JSON generator. + * JsonWriter will not close the generator and only flush it explicitly + * after having written a full Document instance. In other words, JsonWriter + * will <i>not</i> take ownership of the generator. + * + * @param generator + * the output JSON generator + */ + public JsonWriter(JsonGenerator generator) { + this.generator = generator; + } + + /** + * This method will only be called if there is some type which is not + * properly supported in the API, or if something has been changed without + * updating this class. This implementation throws an exception if it is + * reached. + * + * @throws UnsupportedOperationException + * if invoked + */ + @Override + public void write(FieldBase field, FieldValue value) { + throw new UnsupportedOperationException("Serializing " + + value.getClass().getName() + " is not supported."); + } + + @Override + public void write(FieldBase field, Document value) { + try { + fieldNameIfNotNull(field); + generator.writeStartObject(); + // this makes it impossible to refeed directly, not sure what's correct + // perhaps just change to "put"? + generator.writeStringField("id", value.getId().toString()); + generator.writeObjectFieldStart(JsonReader.FIELDS); + for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i + .hasNext();) { + Entry<Field, FieldValue> entry = i.next(); + entry.getValue().serialize(entry.getKey(), this); + } + generator.writeEndObject(); + generator.writeEndObject(); + generator.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public <T extends FieldValue> void write(FieldBase field, Array<T> value) { + try { + fieldNameIfNotNull(field); + generator.writeStartArray(); + for (Iterator<T> i = value.iterator(); i.hasNext();) { + i.next().serialize(null, this); + } + generator.writeEndArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + private void fieldNameIfNotNull(FieldBase field) { + if (field != null) { + try { + generator.writeFieldName(field.getName()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public <K extends FieldValue, V extends FieldValue> void write( + FieldBase field, MapFieldValue<K, V> map) { + fieldNameIfNotNull(field); + try { + generator.writeStartArray(); + for (Map.Entry<K, V> entry : map.entrySet()) { + generator.writeStartObject(); + generator.writeFieldName(JsonReader.MAP_KEY); + entry.getKey().serialize(null, this); + generator.writeFieldName(JsonReader.MAP_VALUE); + entry.getValue().serialize(null, this); + generator.writeEndObject(); + } + generator.writeEndArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void write(FieldBase field, ByteFieldValue value) { + putByte(field, value.getByte()); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, + CollectionFieldValue<T> value) { + fieldNameIfNotNull(field); + try { + generator.writeStartArray(); + for (Iterator<T> i = value.iterator(); i.hasNext();) { + i.next().serialize(null, this); + } + generator.writeEndArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void write(FieldBase field, DoubleFieldValue value) { + putDouble(field, value.getDouble()); + } + + @Override + public void write(FieldBase field, FloatFieldValue value) { + putFloat(field, value.getFloat()); + } + + @Override + public void write(FieldBase field, IntegerFieldValue value) { + putInt(field, value.getInteger()); + } + + @Override + public void write(FieldBase field, LongFieldValue value) { + putLong(field, value.getLong()); + } + + @Override + public void write(FieldBase field, Raw value) { + put(field, value.getByteBuffer()); + } + + @Override + public void write(FieldBase field, PredicateFieldValue value) { + put(field, value.toString()); + } + + @Override + public void write(FieldBase field, StringFieldValue value) { + put(field, value.getString()); + } + + @Override + public void write(FieldBase field, TensorFieldValue value) { + try { + fieldNameIfNotNull(field); + generator.writeStartObject(); + if (value.getTensor().isPresent()) { + Tensor tensor = value.getTensor().get(); + writeTensorDimensions(tensor.dimensions()); + writeTensorCells(tensor.cells()); + } + generator.writeEndObject(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void writeTensorDimensions(Set<String> dimensions) throws IOException { + generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS); + for (String dimension : dimensions) { + generator.writeString(dimension); + } + generator.writeEndArray(); + } + + private void writeTensorCells(Map<TensorAddress, Double> cells) throws IOException { + generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS); + for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) { + generator.writeStartObject(); + writeTensorAddress(cell.getKey()); + generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue()); + generator.writeEndObject(); + } + generator.writeEndArray(); + } + + private void writeTensorAddress(TensorAddress address) throws IOException { + generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS); + for (TensorAddress.Element element : address.elements()) { + generator.writeStringField(element.dimension(), element.label()); + } + generator.writeEndObject(); + } + + @Override + public void write(FieldBase field, Struct value) { + if (value.getDataType() == PositionDataType.INSTANCE) { + put(field, PositionDataType.renderAsString(value)); + return; + } + fieldNameIfNotNull(field); + try { + generator.writeStartObject(); + for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i + .hasNext();) { + Entry<Field, FieldValue> entry = i.next(); + entry.getValue().serialize(entry.getKey(), this); + } + generator.writeEndObject(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void write(FieldBase field, StructuredFieldValue value) { + fieldNameIfNotNull(field); + try { + generator.writeStartObject(); + for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i + .hasNext();) { + Entry<Field, FieldValue> entry = i.next(); + entry.getValue().serialize(entry.getKey(), this); + } + generator.writeEndObject(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public <T extends FieldValue> void write(FieldBase field, + WeightedSet<T> value) { + fieldNameIfNotNull(field); + try { + generator.writeStartObject(); + // entrySet() is deprecated and there is no entry iterator + for (T key : value.keySet()) { + Integer weight = value.get(key); + // key.toString() is according to spec + generator.writeNumberField(key.toString(), weight); + } + generator.writeEndObject(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void write(FieldBase field, AnnotationReference value) { + // not yet implemented, it's not available in XML either + // TODO implement + } + + @Override + public Serializer putByte(FieldBase field, byte value) { + fieldNameIfNotNull(field); + try { + generator.writeNumber(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public Serializer putShort(FieldBase field, short value) { + fieldNameIfNotNull(field); + try { + generator.writeNumber(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public Serializer putInt(FieldBase field, int value) { + fieldNameIfNotNull(field); + try { + generator.writeNumber(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public Serializer putLong(FieldBase field, long value) { + fieldNameIfNotNull(field); + try { + generator.writeNumber(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public Serializer putFloat(FieldBase field, float value) { + fieldNameIfNotNull(field); + try { + generator.writeNumber(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public Serializer putDouble(FieldBase field, double value) { + fieldNameIfNotNull(field); + try { + generator.writeNumber(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public Serializer put(FieldBase field, byte[] value) { + return put(field, ByteBuffer.wrap(value)); + } + + @Override + public Serializer put(FieldBase field, ByteBuffer raw) { + final byte[] data = new byte[raw.remaining()]; + final int origPosition = raw.position(); + + fieldNameIfNotNull(field); + // base64encoder has no encode methods with offset and + // limit, so no use trying to get at the backing array if + // available anyway + raw.get(data); + raw.position(origPosition); + try { + generator.writeString(base64Encoder.encodeToString(data)); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public Serializer put(FieldBase field, String value) { + if (value.length() == 0) { + return this; + } + fieldNameIfNotNull(field); + try { + generator.writeString(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override + public void write(Document document) { + write(null, document); + } + + @Override + public void write(DocumentId id) { + // NOP, fetched from Document + } + + @Override + public void write(DocumentType type) { + // NOP, fetched from Document + } + + /** + * Utility method to easily serialize a single document. + * + * @param document + * the document to be serialized + * @return the input document serialised as UTF-8 encoded JSON + */ + public static byte[] toByteArray(@NonNull Document document) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + JsonWriter writer = new JsonWriter(out); + writer.write(document); + return out.toByteArray(); + } + + /** + * Utility method to easily serialize a single document ID as a remove + * operation. + * + * @param docId + * the document to remove or which has been removed + * @return a document remove operation serialised as UTF-8 encoded JSON for + * the input document ID + */ + public static byte[] documentRemove(@NonNull DocumentId docId) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + JsonGenerator throwAway = jsonFactory.createGenerator(out); + throwAway.writeStartObject(); + throwAway.writeStringField(JsonReader.REMOVE, docId.toString()); + throwAway.writeEndObject(); + throwAway.close(); + } catch (IOException e) { + // Under normal circumstances, nothing here will be triggered + throw new RuntimeException(e); + } + return out.toByteArray(); + } +} diff --git a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java new file mode 100644 index 00000000000..2b210cb2ee5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import com.fasterxml.jackson.core.JsonFactory; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Parser that supports parsing PUT operation and UPDATE operation. + * + * @author dybdahl + */ +public class SingleDocumentParser { + private static final JsonFactory jsonFactory = new JsonFactory(); + private DocumentTypeManager docMan; + + public SingleDocumentParser(DocumentTypeManager docMan) { + this.docMan = docMan; + } + + public VespaXMLFeedReader.Operation parsePut(InputStream inputStream, String docId) { + return parse(inputStream, docId, JsonReader.SupportedOperation.PUT); + } + + public VespaXMLFeedReader.Operation parseUpdate(InputStream inputStream, String docId) { + return parse(inputStream, docId, JsonReader.SupportedOperation.UPDATE); + } + + private VespaXMLFeedReader.Operation parse(InputStream inputStream, String docId, JsonReader.SupportedOperation supportedOperation) { + final JsonReader reader = new JsonReader(docMan, inputStream, jsonFactory); + final DocumentOperation documentOperation = reader.readSingleDocument(supportedOperation, docId); + VespaXMLFeedReader.Operation operation = new VespaXMLFeedReader.Operation(); + try { + inputStream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (supportedOperation == JsonReader.SupportedOperation.PUT) { + operation.setDocument(((DocumentPut) documentOperation).getDocument()); + } else { + operation.setDocumentUpdate((DocumentUpdate) documentOperation); + } + + // (A potentially empty) test-and-set condition is always set by JsonReader + operation.setCondition(documentOperation.getCondition()); + + return operation; + } +} diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java new file mode 100644 index 00000000000..8f3395b989e --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -0,0 +1,195 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.base.Preconditions; + +/** + * Helper class to enable lookahead in the token stream. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +class TokenBuffer { + static final class Token { + final JsonToken token; + final String name; + final String text; + + Token(JsonToken token, String name, String text) { + this.token = token; + this.name = name; + this.text = text; + } + } + + private Deque<Token> buffer; + private int nesting = 0; + + TokenBuffer() { + this(new ArrayDeque<>()); + } + + private TokenBuffer(Deque<Token> buffer) { + this.buffer = buffer; + if (buffer.size() > 0) { + updateNesting(buffer.peekFirst().token); + } + } + + JsonToken next() { + buffer.removeFirst(); + Token t = buffer.peekFirst(); + if (t == null) { + return null; + } + updateNesting(t.token); + return t.token; + } + + JsonToken currentToken() { + return buffer.peekFirst().token; + } + + String currentName() { + return buffer.peekFirst().name; + } + + String currentText() { + return buffer.peekFirst().text; + } + + int size() { + return buffer.size(); + } + + private void add(JsonToken token, String name, String text) { + buffer.addLast(new Token(token, name, text)); + } + + void bufferObject(JsonToken first, JsonParser tokens) { + int localNesting = 0; + JsonToken t = first; + + Preconditions.checkArgument(first == JsonToken.START_OBJECT, + "Expected START_OBJECT, got %s.", t); + if (size() == 0) { + updateNesting(t); + } + localNesting = storeAndPeekNesting(t, localNesting, tokens); + while (localNesting > 0) { + t = nextValue(tokens); + localNesting = storeAndPeekNesting(t, localNesting, tokens); + } + } + + private int storeAndPeekNesting(JsonToken t, int nesting, JsonParser tokens) { + addFromParser(t, tokens); + return nesting + nestingOffset(t); + } + + private int nestingOffset(JsonToken t) { + if (t.isStructStart()) { + return 1; + } else if (t.isStructEnd()) { + return -1; + } else { + return 0; + } + } + + private void addFromParser(JsonToken t, JsonParser tokens) { + try { + add(t, tokens.getCurrentName(), tokens.getText()); + } catch (IOException e) { + // TODO something sane + throw new RuntimeException(e); + } + } + + private JsonToken nextValue(JsonParser tokens) { + try { + return tokens.nextValue(); + } catch (IOException e) { + // TODO something sane + throw new RuntimeException(e); + } + } + + private void updateNesting(JsonToken t) { + nesting += nestingOffset(t); + } + + public int nesting() { + return nesting; + } + + public String dumpContents() { + StringBuilder b = new StringBuilder(); + b.append("[nesting: ").append(nesting()).append("\n"); + for (Token t : buffer) { + b.append("(").append(t.token).append(", \"").append(t.name).append("\", \"").append(t.text).append("\")\n"); + } + b.append("]\n"); + return b.toString(); + } + + public void fastForwardToEndObject() { + JsonToken t = currentToken(); + while (t != JsonToken.END_OBJECT) { + t = next(); + } + } + + TokenBuffer prefetchCurrentElement() { + Deque<Token> copy = new ArrayDeque<>(); + + if (currentToken().isScalarValue()) { + copy.add(buffer.peekFirst()); + } else { + int localNesting = nesting(); + int nestingBarrier = localNesting; + for (Token t : buffer) { + copy.add(t); + localNesting += nestingOffset(t.token); + if (localNesting < nestingBarrier) { + break; + } + } + } + return new TokenBuffer(copy); + } + + Token prefetchScalar(String name) { + int localNesting = nesting(); + int nestingBarrier = localNesting; + Token toReturn = null; + Iterator<Token> i; + + if (name.equals(currentName()) && currentToken().isScalarValue()) { + toReturn = buffer.peekFirst(); + } else { + i = buffer.iterator(); + i.next(); // just ignore the first value, as we know it's not what + // we're looking for, and it's nesting effect is already + // included + while (i.hasNext()) { + Token t = i.next(); + if (localNesting == nestingBarrier && name.equals(t.name) && t.token.isScalarValue()) { + toReturn = t; + break; + } + localNesting += nestingOffset(t.token); + if (localNesting < nestingBarrier) { + break; + } + } + } + return toReturn; + } +} diff --git a/document/src/main/java/com/yahoo/document/json/package-info.java b/document/src/main/java/com/yahoo/document/json/package-info.java new file mode 100644 index 00000000000..85d939f5b18 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Infrastructure for building Vespa documents and feed operations from JSON. + */ +@ExportPackage +package com.yahoo.document.json; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/package-info.java b/document/src/main/java/com/yahoo/document/package-info.java new file mode 100644 index 00000000000..e27bbadacb7 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/select/BucketSelector.java b/document/src/main/java/com/yahoo/document/select/BucketSelector.java new file mode 100644 index 00000000000..96b8e4b617f --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/BucketSelector.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select; + +import com.yahoo.document.BucketIdFactory; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.select.parser.SelectInput; +import com.yahoo.document.select.parser.SelectParser; +import com.yahoo.document.select.parser.TokenMgrError; +import com.yahoo.document.select.simple.SelectionParser; + +/** + * This class is used to find out in which locations a document might be in, if + * it matches a given document selection string. + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class BucketSelector { + + // A local reference to the factory used by the current application. + private BucketIdFactory factory; + + /** + * The bucket selector needs to be instantiated to be used, as it will + * depend on config. + * + * @param factory The bucket factory is needed to get information of how + * bucket ids are put together. + */ + public BucketSelector(BucketIdFactory factory) { + this.factory = factory; + } + + /** + * Get the set of buckets that may contain documents that match the given + * document selection, as long as the document selection does not result in + * an unknown set of buckets. If it does, <code>null</code> will be + * returned. This requires the caller to be aware of the meaning of these + * return values, but also removes the need for redundant space utilization + * when dealing with unknown bucket sets. + * + * @param selector The document selection string + * @return A list of buckets with arbitrary number of location bits set, + * <i>or</i>, <code>null</code> if the document selection resulted + * in an unknown set + * @throws ParseException if <code>selector</code> couldn't be parsed + */ + public BucketSet getBucketList(String selector) throws ParseException { + try { + SelectionParser simple = new SelectionParser(); + if (simple.parse(selector) && (simple.getRemaining().length() == 0)) { + return simple.getNode().getBucketSet(factory); + } else { + SelectParser parser = new SelectParser(new SelectInput(selector)); + return parser.expression().getBucketSet(factory); + } + } catch (TokenMgrError e) { + ParseException t = new ParseException(); + throw (ParseException) t.initCause(e); + } catch (RuntimeException e) { + ParseException t = new ParseException( + "Unexpected exception while parsing '" + selector + "'."); + throw (ParseException) t.initCause(e); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/select/BucketSet.java b/document/src/main/java/com/yahoo/document/select/BucketSet.java new file mode 100644 index 00000000000..e7bb4ac7807 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/BucketSet.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select;
+
+import com.yahoo.document.BucketId;
+
+import java.util.HashSet;
+
+/**
+ * A set of bucket ids covered by a document selector.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class BucketSet extends HashSet<BucketId> {
+
+ /**
+ * Constructs a new bucket set that contains no ids.
+ */
+ public BucketSet() {
+ // empty
+ }
+
+ /**
+ * Constructs a new bucket set that contains a single id.
+ *
+ * @param id The id to add to this as initial value.
+ */
+ public BucketSet(BucketId id) {
+ add(id);
+ }
+
+ /**
+ * Constructs a new bucket set that is a copy of another.
+ *
+ * @param set The set to copy.
+ */
+ public BucketSet(BucketSet set) {
+ this.addAll(set);
+ }
+
+ /**
+ * Returns the intersection between this bucket set and another.
+ *
+ * @param rhs The set to form an intersection with.
+ * @return The intersection.
+ */
+ public BucketSet intersection(BucketSet rhs) {
+ if (rhs == null) {
+ return new BucketSet(this); // The other has all buckets marked, this is the smaller.
+ } else {
+ BucketSet ret = new BucketSet(this);
+ ret.retainAll(rhs);
+ return ret;
+ }
+ }
+
+ /**
+ * Returns the union between this bucket set and another.
+ *
+ * @param rhs The set to form a union with.
+ * @return The union.
+ */
+ public BucketSet union(BucketSet rhs) {
+ if (rhs == null) {
+ return null;
+ } else {
+ BucketSet ret = new BucketSet(this);
+ ret.addAll(rhs);
+ return ret;
+ }
+ }
+
+}
diff --git a/document/src/main/java/com/yahoo/document/select/Context.java b/document/src/main/java/com/yahoo/document/select/Context.java new file mode 100644 index 00000000000..6cf5ab12bbf --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/Context.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select; + +import com.yahoo.document.DocumentOperation; + +import java.util.HashMap; +import java.util.Map; + +public class Context { + private DocumentOperation documentOperation = null; + + public Context(DocumentOperation documentOperation) { + this.documentOperation = documentOperation; + } + + public DocumentOperation getDocumentOperation() { + return documentOperation; + } + + public void setDocumentOperation(DocumentOperation documentOperation) { + this.documentOperation = documentOperation; + } + + public Map<String, Object> getVariables() { + return variables; + } + + public void setVariables(Map<String, Object> variables) { + this.variables = variables; + } + + private Map<String, Object> variables = new HashMap<String, Object>(); + +} diff --git a/document/src/main/java/com/yahoo/document/select/DocumentSelector.java b/document/src/main/java/com/yahoo/document/select/DocumentSelector.java new file mode 100644 index 00000000000..aa26efb0c2d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/DocumentSelector.java @@ -0,0 +1,118 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select; + +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.select.parser.SelectInput; +import com.yahoo.document.select.parser.SelectParser; +import com.yahoo.document.select.parser.TokenMgrError; +import com.yahoo.document.select.rule.ExpressionNode; + +/** + * <p>A document selector is a filter which accepts or rejects documents + * based on their type and content. A document selector has a textual + * representation which is called the <i>Document Selection Language</i></p> + * + * <p>Document selectors are multithread safe.</p> + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class DocumentSelector { + + private ExpressionNode expression; + + /** + * Creates a document selector from a Document Selection Language string + * + * @param selector The string to parse as a selector. + * @throws ParseException Thrown if the string could not be parsed. + */ + public DocumentSelector(String selector) throws ParseException { + SelectInput input = new SelectInput(selector); + try { + SelectParser parser = new SelectParser(input); + expression = parser.expression(); + } catch (TokenMgrError e) { + ParseException t = new ParseException("Tokenization error parsing document selector '" + selector + "'"); + throw (ParseException)t.initCause(e); + } catch (RuntimeException | ParseException e) { + ParseException t = new ParseException("Exception parsing document selector '" + selector + "'"); + throw (ParseException)t.initCause(e instanceof ParseException ? + new ParseException(input.formatException(e.getMessage())) : e); + } + } + + /** + * Returns true if the document referenced by this document operation is accepted by this selector + * + * @param op A document operation + * @return True if the document is accepted. + * @throws RuntimeException if the evaluation enters an illegal state + */ + public Result accepts(DocumentOperation op) { + return accepts(new Context(op)); + } + + /** + * Returns true if the document referenced by this context is accepted by this selector + * + * @param context The context to match in. + * @return True if the document is accepted. + * @throws RuntimeException if the evaluation enters an illegal state + */ + public Result accepts(Context context) { + return Result.toResult(expression.evaluate(context)); + } + + /** + * Returns the list of different variables resulting in a true state for this + * expression. + * + * @param op The document to evaluate. + * @return True if the document is accepted. + * @throws RuntimeException if the evaluation enters an illegal state + */ + public ResultList getMatchingResultList(DocumentOperation op) { + return getMatchingResultList(new Context(op)); + } + + /** + * Returns the list of different variables resulting in a true state for this + * expression. + * + * @param context The context to match in. + * @return True if the document is accepted. + * @throws RuntimeException if the evaluation enters an illegal state + */ + public ResultList getMatchingResultList(Context context) { + return ResultList.toResultList(expression.evaluate(context)); + } + + /** + * Returns this selector as a Document Selection Language string. + * + * @return The selection string. + */ + public String toString() { + return expression.toString(); + } + + /** + * Returns the ordering specification, if any, implied by this document + * selection expression. + * + * @param order The order of the + */ + public OrderingSpecification getOrdering(int order) { + return expression.getOrdering(order); + } + + /** + * Visits the expression tree. + * + * @param visitor The visitor to use. + */ + public void visit(Visitor visitor) { + expression.accept(visitor); + } +} diff --git a/document/src/main/java/com/yahoo/document/select/NowCheckVisitor.java b/document/src/main/java/com/yahoo/document/select/NowCheckVisitor.java new file mode 100644 index 00000000000..50613f531b2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/NowCheckVisitor.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select; + +import com.yahoo.document.select.Visitor; +import com.yahoo.document.select.rule.*; + +/** + * Traverse and check if there exists any now() function in the expression tree. + * + * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + */ + +public class NowCheckVisitor implements Visitor { + private int nowNodeCount = 0; + + public boolean requiresConversion() { + return (nowNodeCount > 0); + } + + public void visit(ArithmeticNode node) { + for (ArithmeticNode.NodeItem item : node.getItems()) { + item.getNode().accept(this); + } + } + + public void visit(AttributeNode node) { + node.getValue().accept(this); + } + + public void visit(ComparisonNode node) { + node.getLHS().accept(this); + node.getRHS().accept(this); + } + + public void visit(DocumentNode node) { + } + + public void visit(EmbracedNode node) { + node.getNode().accept(this); + } + + public void visit(IdNode node) { + } + + public void visit(LiteralNode node) { + } + + public void visit(LogicNode node) { + for (LogicNode.NodeItem item : node.getItems()) { + item.getNode().accept(this); + } + } + + public void visit(NegationNode node) { + node.getNode().accept(this); + } + + public void visit(NowNode node) { + nowNodeCount++; + } + + public void visit(SearchColumnNode node) { + } + + public void visit(VariableNode node) { + } +} diff --git a/document/src/main/java/com/yahoo/document/select/OrderingSpecification.java b/document/src/main/java/com/yahoo/document/select/OrderingSpecification.java new file mode 100644 index 00000000000..3f5a7a58733 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/OrderingSpecification.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select; + +public class OrderingSpecification { + public static int ASCENDING = 0; + public static int DESCENDING = 1; + + public int order; + public long orderingStart; + public short widthBits; + public short divisionBits; + + public OrderingSpecification() { + this(ASCENDING, (long)0, (short)0, (short)0); + } + + public OrderingSpecification(int order) { + this(order, (long)0, (short)0, (short)0); + } + + public OrderingSpecification(int order, long orderingStart, short widthBits, short divisionBits) { + this.order = order; + this.orderingStart = orderingStart; + this.widthBits = widthBits; + this.divisionBits = divisionBits; + } + + public int getOrder() { return order; } + public long getOrderingStart() { return orderingStart; } + public short getWidthBits() { return widthBits; } + public short getDivisionBits() { return divisionBits; } + + public boolean equals(Object other) { + OrderingSpecification o = (OrderingSpecification)other; + if (o == null) return false; + + return (order == o.order && orderingStart == o.orderingStart && widthBits == o.widthBits && divisionBits == o.divisionBits); + } + + public String toString() { + return "O: " + order + " S:" + orderingStart + " W:" + widthBits + " D:" + divisionBits; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/Result.java b/document/src/main/java/com/yahoo/document/select/Result.java new file mode 100644 index 00000000000..3f1fa75d4ef --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/Result.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select;
+
+import com.yahoo.document.select.rule.AttributeNode;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public enum Result {
+
+ /**
+ * Defines all enumeration constants.
+ */
+ TRUE,
+ FALSE,
+ INVALID;
+
+ // Inherit doc from Object.
+ public String toString() {
+ return name().toLowerCase();
+ }
+
+ /**
+ * Inverts the result value to the appropriate value. True → False, False → True and Invalid → Invalid.
+ * @return inverted result
+ */
+ public static Result invert(Result result) {
+ if (result == Result.TRUE) return Result.FALSE;
+ if (result == Result.FALSE) return Result.TRUE;
+ return Result.INVALID;
+ }
+
+ /**
+ * Converts the given object value into an instance of this Result enumeration.
+ *
+ * @param value The object to convert.
+ * @return The corresponding result value.
+ */
+ public static Result toResult(Object value) {
+ if (value == null || value == Result.FALSE || value == Boolean.FALSE ||
+ (Number.class.isInstance(value) && ((Number)value).doubleValue() == 0)) {
+ return Result.FALSE;
+ } else if (value == INVALID) {
+ return Result.INVALID;
+ } else if (value instanceof AttributeNode.VariableValueList) {
+ return ((AttributeNode.VariableValueList)value).isEmpty() ? Result.FALSE : Result.TRUE;
+ } else if (value instanceof ResultList) {
+ return ((ResultList)value).toResult();
+ } else {
+ return Result.TRUE;
+ }
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/ResultList.java b/document/src/main/java/com/yahoo/document/select/ResultList.java new file mode 100644 index 00000000000..e3fc7cadce7 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/ResultList.java @@ -0,0 +1,199 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select; + +import com.yahoo.document.datatypes.FieldPathIteratorHandler; +import com.yahoo.document.select.rule.AttributeNode; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class ResultList { + public static class ResultPair { + ResultPair(FieldPathIteratorHandler.VariableMap var, Result res) { + variables = var; + result = res; + } + + FieldPathIteratorHandler.VariableMap variables; + Result result; + + public FieldPathIteratorHandler.VariableMap getVariables() { return variables; } + public Result getResult() { return result; } + + public String toString() { + return variables.toString() + " => " + result; + } + } + + public static class VariableValue { + public VariableValue(FieldPathIteratorHandler.VariableMap vars, Object value) { + variables = vars; + this.value = value; + } + + FieldPathIteratorHandler.VariableMap variables; + Object value; + + public FieldPathIteratorHandler.VariableMap getVariables() { return variables; } + public Object getValue() { return value; } + + public String toString() { + return variables.toString() + " => " + value; + } + } + + List<ResultPair> results = new ArrayList<ResultPair>(); + + public ResultList() { + } + + public ResultList(Result result) { + add(new FieldPathIteratorHandler.VariableMap(), result); + } + + public void add(FieldPathIteratorHandler.VariableMap variables, Result result) { + results.add(new ResultPair(variables, result)); + } + + public List<ResultPair> getResults() { + return results; + } + + public Result toResult() { + if (results.isEmpty()) { + return Result.FALSE; + } + + boolean foundFalse = false; + + for (ResultPair rp : results) { + if (rp.result == Result.TRUE) { + return Result.TRUE; + } else if (rp.result == Result.FALSE) { + foundFalse = true; + } + } + + if (foundFalse) { + return Result.FALSE; + } else { + return Result.INVALID; + } + } + + boolean combineVariables(FieldPathIteratorHandler.VariableMap output, FieldPathIteratorHandler.VariableMap input) { + // First, verify that all variables are overlapping + for (Map.Entry<String, FieldPathIteratorHandler.IndexValue> entry : output.entrySet()) { + FieldPathIteratorHandler.IndexValue found = input.get(entry.getKey()); + + if (found != null) { + if (!(found.equals(entry.getValue()))) { + return false; + } + } + } + + for (Map.Entry<String, FieldPathIteratorHandler.IndexValue> entry : input.entrySet()) { + FieldPathIteratorHandler.IndexValue found = output.get(entry.getKey()); + + if (found != null) { + if (!(found.equals(entry.getValue()))) { + return false; + } + } + } + + // Ok, variables are overlapping. Add all variables from input to output. + for (Map.Entry<String, FieldPathIteratorHandler.IndexValue> entry : input.entrySet()) { + output.put(entry.getKey(), entry.getValue()); + } + + return true; + } + + public ResultList combineAND(ResultList other) + { + ResultList result = new ResultList(); + + // TODO: optimize + for (ResultPair pair : results) { + for (ResultPair otherPair : other.results) { + FieldPathIteratorHandler.VariableMap varMap = (FieldPathIteratorHandler.VariableMap)pair.variables.clone(); + + if (combineVariables(varMap, otherPair.variables)) { + result.add(varMap, combineAND(pair.result, otherPair.result)); + } + } + } + + return result; + } + + private static Result combineAND(Result lhs, Result rhs) { + if (lhs == Result.TRUE && rhs == Result.TRUE) { + return Result.TRUE; + } + if (lhs == Result.FALSE || rhs == Result.FALSE) { + return Result.FALSE; + } + return Result.INVALID; + } + + public ResultList combineOR(ResultList other) + { + ResultList result = new ResultList(); + + // TODO: optimize + for (ResultPair pair : results) { + for (ResultPair otherPair : other.results) { + FieldPathIteratorHandler.VariableMap varMap = (FieldPathIteratorHandler.VariableMap)pair.variables.clone(); + + if (combineVariables(varMap, otherPair.variables)) { + result.add(varMap, combineOR(pair.result, otherPair.result)); + } + } + } + + return result; + } + + private static Result combineOR(Result lhs, Result rhs) { + if (lhs == Result.TRUE || rhs == Result.TRUE) { + return Result.TRUE; + } + if (lhs == Result.FALSE && rhs == Result.FALSE) { + return Result.FALSE; + } + return Result.INVALID; + } + + /** + * Converts the given object value into a result list, so it can be compared by logical operators. + * + * @param value The object to convert. + * @return The corresponding result value. + */ + public static ResultList toResultList(Object value) { + if (value instanceof ResultList) { + return (ResultList)value; + } else if (value instanceof AttributeNode.VariableValueList) { + ResultList retVal = new ResultList(); + for (VariableValue vv : (AttributeNode.VariableValueList)value) { + retVal.add(vv.getVariables(), Result.TRUE); + } + return retVal; + } else if (value == null || value == Result.FALSE || value == Boolean.FALSE || + (Number.class.isInstance(value) && ((Number)value).doubleValue() == 0)) { + return new ResultList(Result.FALSE); + } else if (value == Result.INVALID) { + return new ResultList(Result.INVALID); + } else { + return new ResultList(Result.TRUE); + } + } + + public String toString() { + return results.toString(); + } +} diff --git a/document/src/main/java/com/yahoo/document/select/Visitor.java b/document/src/main/java/com/yahoo/document/select/Visitor.java new file mode 100644 index 00000000000..a28be4eadaf --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/Visitor.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select; + +import com.yahoo.document.select.rule.*; + +/** + * This interface can be used to create custom visitors for the selection tree. + * + * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + */ + +public interface Visitor { + public void visit(ArithmeticNode node); + public void visit(AttributeNode node); + public void visit(ComparisonNode node); + public void visit(DocumentNode node); + public void visit(EmbracedNode node); + public void visit(IdNode node); + public void visit(LiteralNode node); + public void visit(LogicNode node); + public void visit(NegationNode node); + public void visit(NowNode node); + public void visit(SearchColumnNode node); + public void visit(VariableNode node); +} diff --git a/document/src/main/java/com/yahoo/document/select/convert/NowQueryExpression.java b/document/src/main/java/com/yahoo/document/select/convert/NowQueryExpression.java new file mode 100644 index 00000000000..7041f7b760b --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/convert/NowQueryExpression.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.convert; + +import com.yahoo.document.select.rule.ArithmeticNode; +import com.yahoo.document.select.rule.AttributeNode; +import com.yahoo.document.select.rule.ComparisonNode; + +/** + * Represents a query containing a valid now() expression. The now expression + * is very strict right now, but can be expanded later. + * + * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + */ +public class NowQueryExpression { + private final AttributeNode attribute; + private final ComparisonNode comparison; + private final NowQueryNode now; + + public NowQueryExpression(AttributeNode attribute, ComparisonNode comparison, ArithmeticNode arithmetic) { + this.attribute = attribute; + this.comparison = comparison; + this.now = (arithmetic != null ? new NowQueryNode(arithmetic) : new NowQueryNode(0)); + } + + public String getDocumentType() { + return attribute.getValue().toString(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for(AttributeNode.Item item : attribute.getItems()) { + sb.append(item.getName()).append("."); + } + sb.deleteCharAt(sb.length() - 1); + return sb.toString() + ":" + comparison.getOperator() + now; + } + +} diff --git a/document/src/main/java/com/yahoo/document/select/convert/NowQueryNode.java b/document/src/main/java/com/yahoo/document/select/convert/NowQueryNode.java new file mode 100644 index 00000000000..58b57cc2983 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/convert/NowQueryNode.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.convert; + +import com.yahoo.document.select.rule.ArithmeticNode; + +/** + * Represents the now node in a query expression. + * + * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + */ +public class NowQueryNode { + private final long value; + public NowQueryNode(long value) { + this.value = value; + } + public NowQueryNode(ArithmeticNode node) { + // Assumes that the structure is checked and verified earlier + this.value = Long.parseLong(node.getItems().get(1).getNode().toString()); + } + @Override + public String toString() { + return "now(" + this.value + ")"; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/convert/SelectionExpressionConverter.java b/document/src/main/java/com/yahoo/document/select/convert/SelectionExpressionConverter.java new file mode 100644 index 00000000000..eec14ed7cee --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/convert/SelectionExpressionConverter.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.convert; + +import com.yahoo.document.select.NowCheckVisitor; +import com.yahoo.document.select.Visitor; +import com.yahoo.document.select.rule.*; +import java.util.HashMap; +import java.util.Map; + +/** + * Class which converts a selection tree into a set of queries per document type. + * If unsupported operations are or illegal arguments are encountered, an exception is thrown. + * + * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + */ +public class SelectionExpressionConverter implements Visitor { + + private Map<String, NowQueryExpression> expressionMap = new HashMap<String, NowQueryExpression>(); + + private class BuildState { + public AttributeNode attribute; + public ComparisonNode comparison; + public ArithmeticNode arithmetic; + public NowNode now; + public boolean hasNow() { return now != null; } + } + + private BuildState state; + + private boolean hasNow(ExpressionNode node) { + NowCheckVisitor visitor = new NowCheckVisitor(); + node.accept(visitor); + return visitor.requiresConversion(); + } + + public SelectionExpressionConverter() { + this.state = null; + } + + public Map<String, String> getQueryMap() { + Map<String, String> ret = new HashMap<String, String>(); + for (NowQueryExpression expression : expressionMap.values()) { + ret.put(expression.getDocumentType(), expression.toString()); + } + return ret; + } + + + public void visit(ArithmeticNode node) { + if (state == null ) return; + if (node.getItems().size() > 2) { + throw new IllegalArgumentException("Too many arithmetic operations"); + } + for (ArithmeticNode.NodeItem item : node.getItems()) { + if (item.getOperator() != ArithmeticNode.SUB && item.getOperator() != ArithmeticNode.NOP) { + throw new IllegalArgumentException("Arithmetic operator '" + node.operatorToString(item.getOperator()) + "' is not supported"); + } + } + state.arithmetic = node; + + } + + public void visit(AttributeNode node) { + if (state == null ) return; + if (expressionMap.containsKey(node.getValue().toString())) { + throw new IllegalArgumentException("Specifying multiple document types is not allowed"); + } + for (AttributeNode.Item item : node.getItems()) { + if (item.getType() != AttributeNode.Item.ATTRIBUTE) { + throw new IllegalArgumentException("Only attribute items are supported"); + } + } + state.attribute = node; + } + + public void visit(ComparisonNode node) { + if (state != null) { + throw new IllegalArgumentException("Comparison cannot be done within now expression"); + } + if (!hasNow(node)) { + return; + } + state = new BuildState(); + node.getLHS().accept(this); + node.getRHS().accept(this); + + if (!">".equals(node.getOperator())) { + throw new IllegalArgumentException("Comparison operator '" + node.getOperator() + "' is not supported"); + } + if (!(node.getLHS() instanceof AttributeNode)) { + throw new IllegalArgumentException("Left hand side of comparison must be a document field"); + } + state.comparison = node; + if (state.attribute != null && + state.comparison != null && + (state.arithmetic != null || state.now != null)) { + NowQueryExpression expression = new NowQueryExpression(state.attribute, state.comparison, state.arithmetic); + expressionMap.put(expression.getDocumentType(), expression); + state = null; + } + } + + public void visit(DocumentNode node) { + // Silently ignore + } + + public void visit(EmbracedNode node) { + if (state == null ) return; + throw new UnsupportedOperationException("Grouping is not supported yet."); + } + + public void visit(IdNode node) { + if (state == null ) return; + throw new UnsupportedOperationException("Document id not supported yet."); + } + + public void visit(LiteralNode node) { + if (state == null ) return; + if (!(node.getValue() instanceof Long)) { + throw new IllegalArgumentException("Literal " + node + " is not supported"); + } + } + + public void visit(LogicNode node) { + if (state != null) { + throw new IllegalArgumentException("Logic expressions not supported in now expressions"); + } + for (LogicNode.NodeItem item : node.getItems()) { + item.getNode().accept(this); + } + } + + public void visit(NegationNode node) { + if (state == null ) return; + throw new UnsupportedOperationException("Negation not supported yet."); + } + + public void visit(NowNode node) { + if (state == null ) return; + state.now = node; + } + + public void visit(SearchColumnNode node) { + if (state == null ) return; + throw new UnsupportedOperationException("Searchcolumn not supported yet."); + } + + public void visit(VariableNode node) { + if (state == null ) return; + throw new UnsupportedOperationException("Variables not supported yet."); + } +} diff --git a/document/src/main/java/com/yahoo/document/select/convert/package-info.java b/document/src/main/java/com/yahoo/document/select/convert/package-info.java new file mode 100644 index 00000000000..08e965f317b --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/convert/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.select.convert; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/select/package-info.java b/document/src/main/java/com/yahoo/document/select/package-info.java new file mode 100644 index 00000000000..bbc33bcedf8 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.select; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/select/parser/SelectInput.java b/document/src/main/java/com/yahoo/document/select/parser/SelectInput.java new file mode 100644 index 00000000000..957a038d44a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/parser/SelectInput.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.parser; + +import com.yahoo.javacc.FastCharStream; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class SelectInput extends FastCharStream implements CharStream { + + public SelectInput(String input) { + super(input); + } +} diff --git a/document/src/main/java/com/yahoo/document/select/parser/SelectParserUtils.java b/document/src/main/java/com/yahoo/document/select/parser/SelectParserUtils.java new file mode 100644 index 00000000000..c8458bcb5aa --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/parser/SelectParserUtils.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.parser; + +import com.yahoo.javacc.UnicodeUtilities; + +import java.math.BigInteger; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class SelectParserUtils { + + public static long decodeLong(String str) { + if (str.startsWith("0x") || str.startsWith("0X")) { + str = Long.toString(new BigInteger(str.substring(2), 16).longValue()); + } + return Long.decode(str); + } + + public static String quote(String str, char quote) { + return UnicodeUtilities.quote(str, quote); + } + + public static String unquote(String str) { + return UnicodeUtilities.unquote(str); + } +} diff --git a/document/src/main/java/com/yahoo/document/select/parser/package-info.java b/document/src/main/java/com/yahoo/document/select/parser/package-info.java new file mode 100644 index 00000000000..7f058e86e28 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/parser/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.select.parser; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java new file mode 100644 index 00000000000..2fe4609b4e6 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/ArithmeticNode.java @@ -0,0 +1,209 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.datatypes.NumericFieldValue;
+import com.yahoo.document.select.*;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Stack;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ArithmeticNode implements ExpressionNode {
+
+ public static final int NOP = 0;
+ public static final int ADD = 1;
+ public static final int SUB = 2;
+ public static final int MOD = 3;
+ public static final int DIV = 4;
+ public static final int MUL = 5;
+
+ private final List<NodeItem> items = new ArrayList<NodeItem>();
+
+ public ArithmeticNode() {
+ // empty
+ }
+
+ public ArithmeticNode add(String operator, ExpressionNode node) {
+ items.add(new NodeItem(stringToOperator(operator), node));
+ return this;
+ }
+
+ public List<NodeItem> getItems() {
+ return items;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public Object evaluate(Context context) {
+ StringBuilder ret = null;
+ Stack<ValueItem> buf = new Stack<ValueItem>();
+ for (int i = 0; i < items.size(); ++i) {
+ NodeItem item = items.get(i);
+ Object val = item.node.evaluate(context);
+
+ if (val == null) {
+ throw new IllegalStateException("Null value found!");
+ }
+
+ if (val instanceof AttributeNode.VariableValueList) {
+ AttributeNode.VariableValueList value = (AttributeNode.VariableValueList)val;
+ if (value.size() == 0) {
+ throw new IllegalArgumentException("Can not perform arithmetic on missing field: "
+ + item.node.toString());
+ } else if (value.size() != 1) {
+ throw new IllegalStateException("Arithmetic is only valid for single values.");
+ } else {
+ val = value.get(0).getValue();
+ }
+ }
+
+ if (val instanceof NumericFieldValue) {
+ val = ((NumericFieldValue)val).getNumber();
+ }
+
+ if (val instanceof String) {
+ if (i == 0) {
+ ret = new StringBuilder();
+ }
+ if (ret != null) {
+ ret.append(val);
+ continue;
+ }
+ } else if (Number.class.isInstance(val)) {
+ if (!buf.isEmpty()) {
+ while (buf.peek().operator > item.operator) {
+ popOffTheTop(buf);
+ }
+ }
+ buf.push(new ValueItem(item.operator, (Number)val));
+ continue;
+ }
+ throw new IllegalStateException("Term '" + item.node + " with class " + val.getClass() + "' does not evaluate to a number.");
+ }
+ if (ret != null) {
+ return ret.toString();
+ }
+ while (buf.size() > 1) {
+ popOffTheTop(buf);
+ }
+ return buf.pop().value;
+ }
+
+ private void popOffTheTop(Stack<ValueItem> buf) {
+ ValueItem rhs = buf.pop();
+ ValueItem lhs = buf.pop();
+ switch (rhs.operator) {
+ case ADD:
+ lhs.value = lhs.value.doubleValue() + rhs.value.doubleValue();
+ break;
+ case SUB:
+ lhs.value = lhs.value.doubleValue() - rhs.value.doubleValue();
+ break;
+ case DIV:
+ lhs.value = lhs.value.doubleValue() / rhs.value.doubleValue();
+ break;
+ case MUL:
+ lhs.value = lhs.value.doubleValue() * rhs.value.doubleValue();
+ break;
+ case MOD:
+ lhs.value = lhs.value.longValue() % rhs.value.longValue();
+ break;
+ default:
+ throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported.");
+ }
+ buf.push(lhs);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder ret = new StringBuilder();
+ for (NodeItem item : items) {
+ if (item.operator != NOP) {
+ ret.append(" ").append(operatorToString(item.operator)).append(" ");
+ }
+ ret.append(item.node);
+ }
+ return ret.toString();
+ }
+
+ public String operatorToString(int operator) {
+ switch (operator) {
+ case NOP:
+ return null;
+ case ADD:
+ return "+";
+ case SUB:
+ return "-";
+ case MOD:
+ return "%";
+ case DIV:
+ return "/";
+ case MUL:
+ return "*";
+ default:
+ throw new IllegalStateException("Arithmetic operator " + operator + " not supported.");
+ }
+ }
+
+ private int stringToOperator(String operator) {
+ if (operator == null) {
+ return NOP;
+ } else if (operator.equals("+")) {
+ return ADD;
+ } else if (operator.equals("-")) {
+ return SUB;
+ } else if (operator.equals("%")) {
+ return MOD;
+ } else if (operator.equals("/")) {
+ return DIV;
+ } else if (operator.equals("*")) {
+ return MUL;
+ } else {
+ throw new IllegalStateException("Arithmetic operator '" + operator + "' not supported.");
+ }
+ }
+
+ public OrderingSpecification getOrdering(int order) {
+ return null;
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ private class ValueItem {
+ public int operator;
+ public Number value;
+
+ public ValueItem(int operator, Number value) {
+ this.operator = operator;
+ this.value = value;
+ }
+ }
+
+ public static class NodeItem {
+ private int operator;
+ private ExpressionNode node;
+
+ public NodeItem(int operator, ExpressionNode node) {
+ this.operator = operator;
+ this.node = node;
+ }
+
+ public int getOperator() {
+ return operator;
+ }
+
+ public ExpressionNode getNode() {
+ return node;
+ }
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java new file mode 100644 index 00000000000..048eb70ac94 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java @@ -0,0 +1,205 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.collections.BobHash;
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.select.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AttributeNode implements ExpressionNode {
+
+ private ExpressionNode value;
+ private final List<Item> items = new ArrayList<Item>();
+
+ public AttributeNode(ExpressionNode value, List items) {
+ this.value = value;
+ for (Object obj : items) {
+ if (obj instanceof Item) {
+ this.items.add((Item)obj);
+ } else {
+ throw new IllegalStateException("Can not add an instance of " + obj.getClass().getName() +
+ " as a function item.");
+ }
+ }
+ }
+
+ public ExpressionNode getValue() {
+ return value;
+ }
+
+ public AttributeNode setValue(ExpressionNode value) {
+ this.value = value;
+ return this;
+ }
+
+ public List<Item> getItems() {
+ return items;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public Object evaluate(Context context) {
+ String pos = value.toString();
+ Object obj = value.evaluate(context);
+
+ StringBuilder builder = new StringBuilder();
+ for (Item item : items) {
+ if (obj == null) {
+ throw new IllegalStateException("Can not invoke '" + item + "' on '" + pos + "' because that term " +
+ "evaluated to null.");
+ }
+ if (item.getType() != Item.FUNCTION) {
+ if (builder.length() > 0) {
+ builder.append(".");
+ }
+
+ builder.append(item.getName());
+ } else {
+ if (builder.length() > 0) {
+ obj = evaluateFieldPath(builder.toString(), obj);
+ builder = new StringBuilder();
+ }
+
+ obj = evaluateFunction(item.getName(), obj);
+ }
+
+ pos = pos + "." + item;
+ }
+
+ if (builder.length() > 0) {
+ obj = evaluateFieldPath(builder.toString(), obj);
+ }
+ return obj;
+ }
+
+ public static class VariableValueList extends ArrayList<ResultList.VariableValue> {
+
+ }
+
+ static class IteratorHandler extends FieldPathIteratorHandler {
+ VariableValueList values = new VariableValueList();
+
+ @Override
+ public void onPrimitive(FieldValue fv) {
+ values.add(new ResultList.VariableValue((VariableMap)getVariables().clone(), fv));
+ }
+ }
+
+ private static Object applyFunction(String function, Object value) {
+ if (function.equalsIgnoreCase("abs")) {
+ if (Number.class.isInstance(value)) {
+ Number nValue = (Number)value;
+ if (value instanceof Double) {
+ return nValue.doubleValue() * (nValue.doubleValue() < 0 ? -1 : 1);
+ } else if (value instanceof Float) {
+ return nValue.floatValue() * (nValue.floatValue() < 0 ? -1 : 1);
+ } else if (value instanceof Long) {
+ return nValue.longValue() * (nValue.longValue() < 0 ? -1 : 1);
+ } else if (value instanceof Integer) {
+ return nValue.intValue() * (nValue.intValue() < 0 ? -1 : 1);
+ }
+ }
+ throw new IllegalStateException("Function 'abs' is only available for numerical values.");
+ } else if (function.equalsIgnoreCase("hash")) {
+ return BobHash.hash(value.toString());
+ } else if (function.equalsIgnoreCase("lowercase")) {
+ return value.toString().toLowerCase();
+ } else if (function.equalsIgnoreCase("uppercase")) {
+ return value.toString().toUpperCase();
+ }
+ throw new IllegalStateException("Function '" + function + "' is not supported.");
+ }
+
+ private static Object evaluateFieldPath(String fieldPth, Object value) {
+ if (value instanceof DocumentPut) {
+ final Document doc = ((DocumentPut) value).getDocument();
+ FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth);
+ IteratorHandler handler = new IteratorHandler();
+ doc.iterateNested(fieldPath, 0, handler);
+ return handler.values;
+ } else if (value instanceof DocumentUpdate) {
+ return Result.INVALID;
+ }
+ return Result.FALSE;
+ //throw new IllegalStateException("Attributes are only available for document types for value '" + value + "'. Looking for " + fieldPth);
+ }
+
+ private static Object evaluateFunction(String function, Object value) {
+ if (value instanceof VariableValueList) {
+ VariableValueList retVal = new VariableValueList();
+
+ for (ResultList.VariableValue val : ((VariableValueList)value)) {
+ retVal.add(new ResultList.VariableValue(
+ (FieldPathIteratorHandler.VariableMap)val.getVariables().clone(),
+ applyFunction(function, val.getValue())));
+ }
+
+ return retVal;
+ }
+
+ return applyFunction(function, value);
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder ret = new StringBuilder();
+ ret.append(value);
+ for (Item item : items) {
+ ret.append(".").append(item);
+ }
+ return ret.toString();
+ }
+
+ public OrderingSpecification getOrdering(int order) {
+ return null;
+ }
+
+ public static class Item {
+ public static final int ATTRIBUTE = 0;
+ public static final int FUNCTION = 1;
+
+ private String name;
+ private int type = ATTRIBUTE;
+
+ public Item(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Item setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public int getType() {
+ return type;
+ }
+
+ public Item setType(int type) {
+ this.type = type;
+ return this;
+ }
+
+ @Override public String toString() {
+ return name + (type == FUNCTION ? "()" : "");
+ }
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java new file mode 100644 index 00000000000..b0d5030978e --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java @@ -0,0 +1,435 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.datatypes.FieldPathIteratorHandler;
+import com.yahoo.document.datatypes.NumericFieldValue;
+import com.yahoo.document.idstring.GroupDocIdString;
+import com.yahoo.document.select.*;
+
+import java.util.List;
+import java.util.regex.Pattern;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ComparisonNode implements ExpressionNode {
+
+ // The left- and right-hand-side of this comparison.
+ private ExpressionNode lhs, rhs;
+
+ // The operator string for this.
+ private String operator;
+
+ /**
+ * Constructs a new comparison node.
+ *
+ * @param lhs The left-hand-side of the comparison.
+ * @param operator The comparison operator.
+ * @param rhs The right-hand-side of the comparison.
+ */
+ public ComparisonNode(ExpressionNode lhs, String operator, ExpressionNode rhs) {
+ this.lhs = lhs;
+ this.operator = operator;
+ this.rhs = rhs;
+ }
+
+ /**
+ * Returns the left hand side of this comparison.
+ *
+ * @return The left hand side expression.
+ */
+ public ExpressionNode getLHS() {
+ return lhs;
+ }
+
+ /**
+ * Sets the left hand side of this comparison.
+ *
+ * @param lhs The new left hand side.
+ * @return This, to allow chaining.
+ */
+ public ComparisonNode setLHS(ExpressionNode lhs) {
+ this.lhs = lhs;
+ return this;
+ }
+
+ /**
+ * Returns the comparison operator of this.
+ *
+ * @return The operator.
+ */
+ public String getOperator() {
+ return operator;
+ }
+
+ /**
+ * Sets the comparison operator of this.
+ *
+ * @param operator The operator string.
+ * @return This, to allow chaining.
+ */
+ public ComparisonNode setOperator(String operator) {
+ this.operator = operator;
+ return this;
+ }
+
+ /**
+ * Returns the right hand side of this comparison.
+ *
+ * @return The right hand side expression.
+ */
+ public ExpressionNode getRHS() {
+ return rhs;
+ }
+
+ /**
+ * Sets the right hand side of this comparison.
+ *
+ * @param rhs The new right hand side.
+ * @return This, to allow chaining.
+ */
+ public ComparisonNode setRHS(ExpressionNode rhs) {
+ this.rhs = rhs;
+ return this;
+ }
+
+ public OrderingSpecification getOrdering(IdNode lhs, LiteralNode rhs, String operator, int order) {
+ if (lhs.getWidthBits() == -1 || lhs.getDivisionBits() == -1 || !(rhs.getValue() instanceof Long)) {
+ return null;
+ }
+
+ if (operator.equals("==") || operator.equals("=")) {
+ return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits());
+ }
+
+ if (order == OrderingSpecification.ASCENDING) {
+ if ((operator.equals("<") || operator.equals("<="))) {
+ return new OrderingSpecification(order, 0, lhs.getWidthBits(), lhs.getDivisionBits());
+ }
+ if (operator.equals(">")) {
+ return new OrderingSpecification(order, (Long)rhs.getValue() + 1, lhs.getWidthBits(), lhs.getDivisionBits());
+ }
+ if (operator.equals(">=")) {
+ return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits());
+ }
+ } else {
+ if (operator.equals("<")) {
+ return new OrderingSpecification(order, (Long)rhs.getValue() - 1, lhs.getWidthBits(), lhs.getDivisionBits());
+ }
+ if (operator.equals("<=")) {
+ return new OrderingSpecification(order, (Long)rhs.getValue(), lhs.getWidthBits(), lhs.getDivisionBits());
+ }
+ }
+ return null;
+ }
+
+ public OrderingSpecification getOrdering(int order) {
+ if (lhs instanceof IdNode && rhs instanceof LiteralNode) {
+ return getOrdering((IdNode)lhs, (LiteralNode)rhs, operator, order);
+ } else if (rhs instanceof IdNode && lhs instanceof LiteralNode) {
+ return getOrdering((IdNode)rhs, (LiteralNode)rhs, operator, order);
+ }
+
+ return null;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ if (operator.equals("==") || operator.equals("=")) {
+ if (lhs instanceof IdNode && rhs instanceof LiteralNode) {
+ return compare(factory, (IdNode)lhs, (LiteralNode)rhs, operator);
+ } else if (rhs instanceof IdNode && lhs instanceof LiteralNode) {
+ return compare(factory, (IdNode)rhs, (LiteralNode)lhs, operator);
+ } else if (lhs instanceof SearchColumnNode && rhs instanceof LiteralNode) {
+ return compare(factory, (SearchColumnNode)lhs, (LiteralNode)rhs);
+ } else if (rhs instanceof SearchColumnNode && lhs instanceof LiteralNode) {
+ return compare(factory, (SearchColumnNode)rhs, (LiteralNode)lhs);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Compares a search column node with a literal node.
+ *
+ * @param factory The bucket id factory used.
+ * @param node The search column node.
+ * @param literal The literal node to compare to.
+ * @return The bucket set containing the buckets covered.
+ */
+ private BucketSet compare(BucketIdFactory factory, SearchColumnNode node, LiteralNode literal) {
+ Object value = literal.getValue();
+ int bucketCount = (int) Math.pow(2, 16);
+ if (value instanceof Long) {
+ BucketSet ret = new BucketSet();
+ for (int i = 0; i < bucketCount; i++) {
+ BucketId id = new BucketId(16, i);
+ if ((Long)value == node.getDistribution().getColumn(id)) {
+ ret.add(new BucketId(16, i));
+ }
+ }
+ return ret;
+ }
+ return null;
+ }
+
+ private BucketSet compare(BucketIdFactory factory, IdNode id, LiteralNode literal, String operator) {
+ String field = id.getField();
+ Object value = literal.getValue();
+ if (field == null) {
+ if (value instanceof String) {
+ String name = (String)value;
+ if ((operator.equals("=") && name.contains("*")) ||
+ (operator.equals("=~") && ((name.contains("*") || name.contains("?")))))
+ {
+ return null; // no idea
+ }
+ return new BucketSet(factory.getBucketId(new DocumentId(name)));
+ }
+ } else if (field.equalsIgnoreCase("user")) {
+ if (value instanceof Long) {
+ return new BucketSet(new BucketId(factory.getLocationBitCount(), (Long)value));
+ }
+ } else if (field.equalsIgnoreCase("group")) {
+ if (value instanceof String) {
+ String name = (String)value;
+ if ((operator.equals("=") && name.contains("*")) ||
+ (operator.equals("=~") && ((name.contains("*") || name.contains("?")))))
+ {
+ return null; // no idea
+ }
+ return new BucketSet(new BucketId(factory.getLocationBitCount(), new GroupDocIdString("", name, "").getLocation()));
+ }
+ } else if (field.equalsIgnoreCase("bucket")) {
+ if (value instanceof Long) {
+ return new BucketSet(new BucketId((Long)value));
+ }
+ }
+ return null;
+ }
+
+ // Inherit doc from Node.
+ public Object evaluate(Context context) {
+ Object oLeft = lhs.evaluate(context);
+ Object oRight = rhs.evaluate(context);
+ if (oLeft == null && oRight == null) {
+ return new ResultList(Result.TRUE);
+ }
+ if (oLeft == Result.INVALID || oRight == Result.INVALID) {
+ return new ResultList(Result.INVALID);
+ }
+ if (oLeft instanceof AttributeNode.VariableValueList && oRight instanceof AttributeNode.VariableValueList) {
+ if (operator.equals("==")) {
+ return evaluateListsTrue((AttributeNode.VariableValueList)oLeft, (AttributeNode.VariableValueList)oRight);
+ } else if (operator.equals("!=")) {
+ return evaluateListsFalse((AttributeNode.VariableValueList)oLeft, (AttributeNode.VariableValueList)oRight);
+ } else {
+ return new ResultList(Result.INVALID);
+ }
+ } else if (oLeft instanceof AttributeNode.VariableValueList) {
+ return evaluateListAndSingle((AttributeNode.VariableValueList)oLeft, oRight);
+ } else if (oRight instanceof AttributeNode.VariableValueList) {
+ return evaluateListAndSingle((AttributeNode.VariableValueList)oRight, oLeft);
+ }
+ return new ResultList(evaluateBool(oLeft, oRight));
+ }
+
+ public ResultList evaluateListsTrue(AttributeNode.VariableValueList lhs, AttributeNode.VariableValueList rhs) {
+ if (lhs.size() != rhs.size()) {
+ return new ResultList(Result.FALSE);
+ }
+
+ for (int i = 0; i < lhs.size(); i++) {
+ if (!lhs.get(i).getVariables().equals(rhs.get(i).getVariables())) {
+ return new ResultList(Result.FALSE);
+ }
+
+ if (evaluateEquals(lhs.get(i).getValue(), rhs.get(i).getValue()) == Result.FALSE) {
+ return new ResultList(Result.FALSE);
+ }
+ }
+
+ return new ResultList(Result.TRUE);
+ }
+
+ public ResultList evaluateListsFalse(AttributeNode.VariableValueList lhs, AttributeNode.VariableValueList rhs) {
+ ResultList lst = evaluateListsTrue(lhs, rhs);
+ if (lst.toResult() == Result.TRUE) {
+ return new ResultList(Result.FALSE);
+ } else if (lst.toResult() == Result.FALSE) {
+ return new ResultList(Result.TRUE);
+ } else {
+ return lst;
+ }
+ }
+
+ public ResultList evaluateListAndSingle(AttributeNode.VariableValueList lhs, Object rhs) {
+ if (rhs == null && lhs == null) {
+ return new ResultList(Result.TRUE);
+ }
+
+ if (rhs == null || lhs == null) {
+ return new ResultList(Result.FALSE);
+ }
+
+ ResultList retVal = new ResultList();
+ for (int i = 0; i < lhs.size(); i++) {
+ Result result = evaluateBool(lhs.get(i).getValue(), rhs);
+ retVal.add((FieldPathIteratorHandler.VariableMap)lhs.get(i).getVariables().clone(), result);
+ }
+
+ return retVal;
+ }
+
+ /**
+ * Evaluate this expression on two operands, given that they are not invalid.
+ *
+ * @param lhs Left hand side of operation.
+ * @param rhs Right hand side of operation.
+ * @return The evaluation result.
+ */
+ private Result evaluateBool(Object lhs, Object rhs) {
+ if (operator.equals("==")) {
+ return evaluateEquals(lhs, rhs);
+ } else if (operator.equals("!=")) {
+ return Result.invert(evaluateEquals(lhs, rhs));
+ } else if (operator.equals("<") || operator.equals("<=") ||
+ operator.equals(">") || operator.equals(">=")) {
+ return evaluateNumber(lhs, rhs);
+ } else if (operator.equals("=~") || operator.equals("=")) {
+ return evaluateString(lhs, rhs);
+ }
+ throw new IllegalStateException("Comparison operator '" + operator + "' is not supported.");
+ }
+
+ /**
+ * Compare two operands for equality.
+ *
+ * @param lhs Left hand side of operation.
+ * @param rhs Right hand side of operation.
+ * @return Wether or not the two operands are equal.
+ */
+ private Result evaluateEquals(Object lhs, Object rhs) {
+ if (lhs == null || rhs == null) {
+ return Result.toResult(lhs == rhs);
+ }
+
+ double a = getAsNumber(lhs);
+ double b = getAsNumber(rhs);
+ if (Double.isNaN(a) || Double.isNaN(b)) {
+ return Result.toResult(lhs.toString().equals(rhs.toString()));
+ }
+ return Result.toResult(a == b); // Ugh, comparing doubles? Should be converted to long value perhaps...
+ }
+
+ private double getAsNumber(Object value) {
+ if (value instanceof Number) {
+ return ((Number)value).doubleValue();
+ } else if (value instanceof NumericFieldValue) {
+ return getAsNumber(((NumericFieldValue)value).getNumber());
+ } else {
+ return Double.NaN; //new IllegalStateException("Term '" + value + "' (" + value.getClass() + ") does not evaluate to a number.");
+ }
+ }
+
+ /**
+ * Evalutes the value of this term over a document, given that both operands must be numbers.
+ *
+ * @param lhs Left hand side of operation.
+ * @param rhs Right hand side of operation.
+ * @return The evaluation result.
+ */
+ private Result evaluateNumber(Object lhs, Object rhs) {
+ double a = getAsNumber(lhs);
+ double b = getAsNumber(rhs);
+ if (Double.isNaN(a) || Double.isNaN(b)) {
+ return Result.INVALID;
+ }
+ if (operator.equals("<")) {
+ return Result.toResult(a < b);
+ } else if (operator.equals("<=")) {
+ return Result.toResult(a <= b);
+ } else if (operator.equals(">")) {
+ return Result.toResult(a > b);
+ } else {
+ return Result.toResult(a >= b);
+ }
+ }
+
+ /**
+ * Evalutes the value of this term over a document, given that both operands must be strings.
+ *
+ * @param lhs Left hand side of operation.
+ * @param rhs Right hand side of operation.
+ * @return The evaluation result.
+ */
+ private Result evaluateString(Object lhs, Object rhs) {
+ String left = "" + lhs; // Allows null objects to evaluate to string.
+ String right = "" + rhs;
+ if (operator.equals("=~")) {
+ return Result.toResult(Pattern.compile(right).matcher(left).find());
+ } else {
+ return Result.toResult(Pattern.compile(globToRegex(right)).matcher(left).find());
+ }
+ }
+
+ /**
+ * Converts a glob pattern to a corresponding regular expression string.
+ *
+ * @param glob The glob pattern.
+ * @return The regex string.
+ */
+ private String globToRegex(String glob) {
+ StringBuilder ret = new StringBuilder();
+ ret.append("^");
+ for (int i = 0; i < glob.length(); i++) {
+ ret.append(globToRegex(glob.charAt(i)));
+ }
+ ret.append("$");
+
+ return ret.toString();
+ }
+
+ /**
+ * Converts a single character in a glob expression to the corresponding regular expression string.
+ *
+ * @param glob The glob character.
+ * @return The regex string.
+ */
+ private String globToRegex(char glob) {
+ switch (glob) {
+ case'*':
+ return ".*";
+ case'?':
+ return ".";
+ case'^':
+ case'$':
+ case'|':
+ case'{':
+ case'}':
+ case'(':
+ case')':
+ case'[':
+ case']':
+ case'\\':
+ case'+':
+ case'.':
+ return "\\" + glob;
+ default:
+ return "" + glob;
+ }
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ // Inherit doc from Object.
+ @Override
+ public String toString() {
+ return lhs + " " + operator + " " + rhs;
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java new file mode 100644 index 00000000000..c071e6674aa --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/DocumentNode.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.*;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DocumentNode implements ExpressionNode {
+
+ private String type;
+
+ public DocumentNode(String type) {
+ this.type = type;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public DocumentNode setType(String type) {
+ this.type = type;
+ return this;
+ }
+
+ @Override
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ @Override
+ public Object evaluate(Context context) {
+ return evaluate(context.getDocumentOperation());
+ }
+
+ public Object evaluate(DocumentOperation op) {
+ DocumentType doct;
+ if (op instanceof DocumentPut) {
+ doct = ((DocumentPut)op).getDocument().getDataType();
+ } else if (op instanceof DocumentUpdate) {
+ doct = ((DocumentUpdate)op).getDocumentType();
+ } else {
+ throw new IllegalStateException("Document class '" + op.getClass().getName() + "' is not supported.");
+ }
+ return doct.isA(this.type) ? op : Boolean.FALSE;
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public String toString() {
+ return type;
+ }
+
+ @Override
+ public OrderingSpecification getOrdering(int order) {
+ return null;
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java b/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java new file mode 100644 index 00000000000..2a74a5af619 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/EmbracedNode.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class EmbracedNode implements ExpressionNode {
+
+ private ExpressionNode node;
+
+ public EmbracedNode(ExpressionNode node) {
+ this.node = node;
+ }
+
+ public ExpressionNode getNode() {
+ return node;
+ }
+
+ public EmbracedNode setNode(ExpressionNode node) {
+ this.node = node;
+ return this;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ public Object evaluate(Context context) {
+ return node.evaluate(context);
+ }
+
+ @Override
+ public String toString() {
+ return "(" + node + ")";
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ public OrderingSpecification getOrdering(int order) {
+ return null;
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java b/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java new file mode 100644 index 00000000000..ba51ec840d4 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/ExpressionNode.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * This is the interface of all expression nodes. It declares the methods requires by all expression nodes to maintain
+ * a working document selector language.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface ExpressionNode {
+
+ /**
+ * Evaluate the content of this node based on document object, and return that value.
+ *
+ * @param doc The document to evaluate over.
+ * @return The value of this.
+ */
+ public Object evaluate(Context doc);
+
+ /**
+ * Returns the set of bucket ids covered by this node.
+ *
+ * @param factory The factory used by the current application.
+ */
+ public BucketSet getBucketSet(BucketIdFactory factory);
+
+ /**
+ * If this document selection implies a specific ordering (using the orderdoc scheme),
+ * return that specification.
+ *
+ * @param order The order in which we are looking to traverse the ordering (ASCENDING or DESCENDING)
+ */
+ public OrderingSpecification getOrdering(int order);
+
+ /**
+ * Perform visitation of this node.
+ *
+ * @param visitor The visitor that wishes to visit the node.
+ */
+ public void accept(Visitor visitor);
+
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/IdNode.java b/document/src/main/java/com/yahoo/document/select/rule/IdNode.java new file mode 100644 index 00000000000..f12ecbb752b --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/IdNode.java @@ -0,0 +1,108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.DocumentId;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.*;
+import com.yahoo.document.idstring.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IdNode implements ExpressionNode {
+
+ private String field;
+ private short widthBits = -1;
+ private short divisionBits = -1;
+
+ public IdNode() {
+ // empty
+ }
+
+ public String getField() {
+ return field;
+ }
+
+ public IdNode setField(String field) {
+ this.field = field;
+ return this;
+ }
+
+ public IdNode setWidthBits(short widthBits) {
+ this.widthBits = widthBits;
+ return this;
+ }
+
+ public IdNode setDivisionBits(short divisionBits) {
+ this.divisionBits = divisionBits;
+ return this;
+ }
+
+ public short getWidthBits() {
+ return widthBits;
+ }
+
+ public short getDivisionBits() {
+ return divisionBits;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ public OrderingSpecification getOrdering(int ordering) {
+ return null;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public Object evaluate(Context context) {
+ DocumentId id = context.getDocumentOperation().getId();
+ if (id == null) {
+ throw new IllegalStateException("Document has no identifier.");
+ }
+ if (field == null) {
+ return id.toString();
+ } else if (field.equalsIgnoreCase("scheme")) {
+ return id.getScheme().getType().toString();
+ } else if (field.equalsIgnoreCase("namespace")) {
+ return id.getScheme().getNamespace();
+ } else if (field.equalsIgnoreCase("specific")) {
+ return id.getScheme().getNamespaceSpecific();
+ } else if (field.equalsIgnoreCase("group")) {
+ if (id.getScheme().hasGroup()) {
+ return id.getScheme().getGroup();
+ }
+ throw new IllegalStateException("Group identifier is null.");
+ } else if (field.equalsIgnoreCase("user")) {
+ if (id.getScheme().hasNumber()) {
+ return id.getScheme().getNumber();
+ }
+ throw new IllegalStateException("User identifier is null.");
+ } else if (field.equalsIgnoreCase("type")) {
+ if (id.getScheme().hasDocType()) {
+ return id.getScheme().getDocType();
+ }
+ throw new IllegalStateException("Document id doesn't have doc type.");
+ } else if (field.equalsIgnoreCase("order")) {
+ if (id.getScheme() instanceof OrderDocIdString) {
+ OrderDocIdString ods = (OrderDocIdString)id.getScheme();
+ if (ods.getWidthBits() == widthBits && ods.getDivisionBits() == divisionBits) {
+ return ods.getOrdering();
+ }
+ }
+ } else{
+ throw new IllegalStateException("Identifier field '" + field + "' is not supported.");
+ }
+ return null;
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public String toString() {
+ return "id" + (field != null ? "." + field : "") + (widthBits != -1 ? "(" + widthBits + "," + divisionBits + ")" : "");
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java b/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java new file mode 100644 index 00000000000..ae0d640d471 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/LiteralNode.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+import com.yahoo.document.select.parser.SelectParserUtils;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LiteralNode implements ExpressionNode {
+
+ private Object value;
+
+ public LiteralNode(Object value) {
+ this.value = value;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+
+ public LiteralNode setValue(Object value) {
+ this.value = value;
+ return this;
+ }
+
+ @Override
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ @Override
+ public Object evaluate(Context context) {
+ return value;
+ }
+
+ @Override
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public String toString() {
+ if (value == null) {
+ return "null";
+ } else if (value instanceof String) {
+ return SelectParserUtils.quote((String)value, '"');
+ } else {
+ return value.toString();
+ }
+ }
+
+ @Override
+ public OrderingSpecification getOrdering(int order) {
+ return null;
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java b/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java new file mode 100644 index 00000000000..145c655fae2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/LogicNode.java @@ -0,0 +1,316 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.ResultList;
+import com.yahoo.document.select.Visitor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Stack;
+
+/**
+ * This class defines a logical expression of nodes. This implementation uses a stack to evaluate its content as to
+ * avoid deep recursions when building the parse tree.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LogicNode implements ExpressionNode {
+
+ // A no-op value is defined for completeness.
+ public static final int NOP = 0;
+
+ // The OR operator has lower precedence than AND.
+ public static final int OR = 1;
+
+ // The AND operator has the highest precedence.
+ public static final int AND = 2;
+
+ // The items contained in this.
+ private final List<NodeItem> items = new ArrayList<NodeItem>();
+
+ /**
+ * Construct an empty logic expression.
+ */
+ public LogicNode() {
+ // empty
+ }
+
+ public List<NodeItem> getItems() {
+ return items;
+ }
+
+ /**
+ * Adds an (operator, node) pair to this expression.
+ *
+ * @param operator The operator that combines the previous with the node given.
+ * @param node The node to add to this.
+ * @return This, to allow chaining.
+ */
+ public LogicNode add(String operator, ExpressionNode node) {
+ items.add(new LogicNode.NodeItem(stringToOperator(operator), node));
+ return this;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ Stack<BucketItem> buf = new Stack<>();
+ for (NodeItem item : items) {
+ if (!buf.isEmpty()) {
+ while (buf.peek().operator > item.operator) {
+ combineBuckets(buf);
+ }
+ }
+ buf.push(new BucketItem(item.operator, item.node.getBucketSet(factory)));
+ }
+ while (buf.size() > 1) {
+ combineBuckets(buf);
+ }
+ return buf.pop().buckets;
+ }
+
+ public OrderingSpecification getOrdering(int order) {
+ Stack<OrderingItem> buf = new Stack<>();
+ for (NodeItem item : items) {
+ if (!buf.isEmpty()) {
+ while (buf.peek().operator > item.operator) {
+ pickOrdering(buf);
+ }
+ }
+ buf.push(new OrderingItem(item.operator, item.node.getOrdering(order)));
+ }
+ while (buf.size() > 1) {
+ pickOrdering(buf);
+ }
+ return buf.pop().ordering;
+ }
+
+ private OrderingSpecification pickOrdering(OrderingSpecification a, OrderingSpecification b, boolean isAnd) {
+ if (a.getWidthBits() == b.getWidthBits() && a.getDivisionBits() == b.getDivisionBits() && a.getOrder() == b.getOrder()) {
+ if ((a.getOrder() == OrderingSpecification.ASCENDING && isAnd) ||
+ (a.getOrder() == OrderingSpecification.DESCENDING && !isAnd)) {
+ return new OrderingSpecification(a.getOrder(), Math.max(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits());
+ } else {
+ return new OrderingSpecification(a.getOrder(), Math.min(a.getOrderingStart(), b.getOrderingStart()), b.getWidthBits(), a.getDivisionBits());
+ }
+ }
+ return null;
+ }
+
+ private void pickOrdering(Stack<OrderingItem> buf) {
+ OrderingItem rhs = buf.pop();
+ OrderingItem lhs = buf.pop();
+ switch (rhs.operator) {
+ case AND:
+ if (lhs.ordering == null) {
+ lhs.ordering = rhs.ordering;
+ } else if (rhs.ordering == null) {
+ // empty
+ } else {
+ lhs.ordering = pickOrdering(lhs.ordering, rhs.ordering, true);
+ }
+ break;
+ case OR:
+ if (lhs.ordering != null && rhs.ordering != null) {
+ lhs.ordering = pickOrdering(lhs.ordering, rhs.ordering, false);
+ } else {
+ lhs.ordering = null;
+ }
+ break;
+ default:
+ lhs.ordering = null;
+ }
+ buf.push(lhs);
+ }
+
+ /**
+ * Combines the top two items of the given stack using the operator of the second.
+ *
+ * @param buf The stack of bucket items.
+ */
+ private void combineBuckets(Stack<BucketItem> buf) {
+ BucketItem rhs = buf.pop();
+ BucketItem lhs = buf.pop();
+ switch (rhs.operator) {
+ case AND:
+ if (lhs.buckets == null) {
+ lhs.buckets = rhs.buckets;
+ } else if (rhs.buckets == null) {
+ // empty
+ } else {
+ lhs.buckets = lhs.buckets.intersection(rhs.buckets);
+ }
+ break;
+ case OR:
+ if (lhs.buckets == null) {
+ // empty
+ } else if (rhs.buckets == null) {
+ lhs.buckets = null;
+ } else {
+ lhs.buckets = lhs.buckets.union(rhs.buckets);
+ }
+ break;
+ default:
+ throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported.");
+ }
+ buf.push(lhs);
+ }
+
+ // Inherit doc from ExpressionNode.
+ @Override
+ public Object evaluate(Context context) {
+ Stack<ValueItem> buf = new Stack<>();
+ for (NodeItem item : items) {
+ if ( ! buf.isEmpty()) {
+ while (buf.peek().operator > item.operator) {
+ combineValues(buf);
+ }
+ }
+
+ buf.push(new ValueItem(item.operator, ResultList.toResultList(item.node.evaluate(context))));
+ }
+ while (buf.size() > 1) {
+ combineValues(buf);
+ }
+ return buf.pop().value;
+ }
+
+ /**
+ * Combines the top two items of the given stack using the operator of the second.
+ *
+ * @param buf The stack of values.
+ */
+ private void combineValues(Stack<ValueItem> buf) {
+ ValueItem rhs = buf.pop();
+ ValueItem lhs = buf.pop();
+
+ switch (rhs.operator) {
+ case AND:
+ buf.push(new ValueItem(lhs.operator, lhs.value.combineAND(rhs.value)));
+ break;
+ case OR:
+ buf.push(new ValueItem(lhs.operator, lhs.value.combineOR(rhs.value)));
+ break;
+ default:
+ throw new IllegalStateException("Arithmetic operator " + rhs.operator + " not supported.");
+ }
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ // Inherit doc from Object.
+ @Override
+ public String toString() {
+ StringBuilder ret = new StringBuilder();
+ for (LogicNode.NodeItem item : items) {
+ if (item.operator != NOP) {
+ ret.append(" ").append(operatorToString(item.operator)).append(" ");
+ }
+ ret.append(item.node);
+ }
+ return ret.toString();
+ }
+
+ /**
+ * Converts the given operator index to a string representation.
+ *
+ * @param operator The operator index to convert.
+ * @return The string representation.
+ */
+ public String operatorToString(int operator) {
+ switch (operator) {
+ case NOP:
+ return null;
+ case OR:
+ return "or";
+ case AND:
+ return "and";
+ default:
+ throw new IllegalStateException("Logical operator " + operator + " not supported.");
+ }
+ }
+
+ /**
+ * Converts the given operator string to a corresponding operator index. This is necessary to perform a stack
+ * traversal of logic expression.
+ *
+ * @param operator The operator to convert.
+ * @return The corresponding index.
+ */
+ private int stringToOperator(String operator) {
+ if (operator == null) {
+ return NOP;
+ } else if (operator.equalsIgnoreCase("or")) {
+ return OR;
+ } else if (operator.equalsIgnoreCase("and")) {
+ return AND;
+ } else {
+ throw new IllegalStateException("Logical operator '" + operator + "' not supported.");
+ }
+ }
+
+ /**
+ * Private class to store results in a stack.
+ */
+ private final class ValueItem {
+ private int operator;
+ private ResultList value;
+
+ public ValueItem(int operator, ResultList value) {
+ this.operator = operator;
+ this.value = value;
+ }
+ }
+
+ /**
+ * Private class to store bucket sets in a stack.
+ */
+ private final class BucketItem {
+ private int operator;
+ private BucketSet buckets;
+
+ public BucketItem(int operator, BucketSet buckets) {
+ this.operator = operator;
+ this.buckets = buckets;
+ }
+ }
+
+ /**
+ * Private class to store ordering expressions in a stack.
+ */
+ private final class OrderingItem {
+ private int operator;
+ private OrderingSpecification ordering;
+
+ public OrderingItem(int operator, OrderingSpecification orderSpec) {
+ this.operator = operator;
+ this.ordering = orderSpec;
+ }
+ }
+
+ /**
+ * Private class to store expression nodes in a stack.
+ */
+ public final class NodeItem {
+ private int operator;
+ private ExpressionNode node;
+
+ public NodeItem(int operator, ExpressionNode node) {
+ this.operator = operator;
+ this.node = node;
+ }
+
+ public int getOperator() {
+ return operator;
+ }
+
+ public ExpressionNode getNode() {
+ return node;
+ }
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java b/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java new file mode 100644 index 00000000000..2b85bc3eee6 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/NegationNode.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Result;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NegationNode implements ExpressionNode {
+
+ private ExpressionNode node;
+
+ public NegationNode(ExpressionNode node) {
+ this.node = node;
+ }
+
+ public ExpressionNode getNode() {
+ return node;
+ }
+
+ public NegationNode setNode(ExpressionNode node) {
+ this.node = node;
+ return this;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public Object evaluate(Context context) {
+ return Result.invert(Result.toResult(node.evaluate(context)));
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public String toString() {
+ return "not " + node;
+ }
+
+ public OrderingSpecification getOrdering(int order) {
+ return null;
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/NowNode.java b/document/src/main/java/com/yahoo/document/select/rule/NowNode.java new file mode 100644 index 00000000000..600dbe536e4 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/NowNode.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.*;
+
+/**
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class NowNode implements ExpressionNode {
+
+ // Inherit doc from ExpressionNode.
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public Object evaluate(Context context) {
+ Object ret = System.currentTimeMillis() / 1000;
+ return ret;
+ }
+
+ @Override
+ public String toString() {
+ return "now()";
+ }
+
+ public OrderingSpecification getOrdering(int order) {
+ return null;
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java b/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java new file mode 100644 index 00000000000..c63197ae3a4 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/SearchColumnNode.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketDistribution;
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SearchColumnNode implements ExpressionNode {
+
+ private int field;
+ private BucketIdFactory factory = new BucketIdFactory(); // why is this not an abstract class?
+ private BucketDistribution distribution;
+
+ public SearchColumnNode() {
+ setField(0);
+ }
+
+ public int getField() {
+ return field;
+ }
+
+ public SearchColumnNode setField(int field) {
+ distribution = new BucketDistribution(this.field = field, 16);
+ return this;
+ }
+
+ public BucketDistribution getDistribution() {
+ return distribution;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ // Inherit doc from ExpressionNode.
+ public Object evaluate(Context context) {
+ return distribution.getColumn(factory.getBucketId(context.getDocumentOperation().getId()));
+ }
+
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public String toString() {
+ return "searchcolumn." + field;
+ }
+
+ public OrderingSpecification getOrdering(int order) {
+ return null;
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java b/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java new file mode 100644 index 00000000000..77c462674db --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/VariableNode.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.rule;
+
+import com.yahoo.document.BucketIdFactory;
+import com.yahoo.document.select.BucketSet;
+import com.yahoo.document.select.Context;
+import com.yahoo.document.select.OrderingSpecification;
+import com.yahoo.document.select.Visitor;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class VariableNode implements ExpressionNode {
+
+ private String value;
+
+ public VariableNode(String value) {
+ this.value = value;
+ }
+
+ public Object getValue() {
+ return value;
+ }
+
+ public VariableNode setValue(String value) {
+ this.value = value;
+ return this;
+ }
+
+ @Override
+ public BucketSet getBucketSet(BucketIdFactory factory) {
+ return null;
+ }
+
+ @Override
+ public Object evaluate(Context context) {
+ Object o = context.getVariables().get(value);
+ if (o == null) {
+ throw new IllegalArgumentException("Variable " + value + " was not set in the variable list");
+ }
+ return o;
+ }
+
+ @Override
+ public void accept(Visitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
+ public String toString() {
+ return "$" + value;
+ }
+
+ @Override
+ public OrderingSpecification getOrdering(int order) {
+ return null;
+ }
+}
diff --git a/document/src/main/java/com/yahoo/document/select/rule/package-info.java b/document/src/main/java/com/yahoo/document/select/rule/package-info.java new file mode 100644 index 00000000000..b5268393037 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/rule/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.select.rule; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/select/simple/IdSpecParser.java b/document/src/main/java/com/yahoo/document/select/simple/IdSpecParser.java new file mode 100644 index 00000000000..b444c2d5ac2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/simple/IdSpecParser.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.simple; + +import com.yahoo.document.select.rule.IdNode; + +/** + * @author balder + */ +public class IdSpecParser extends Parser { + private IdNode id; + public IdNode getId() { return id; } + public boolean isUserSpec() { return "user".equals(id.getField()); } + public boolean parse(CharSequence s) { + boolean retval = false; + int pos = eatWhite(s); + if (pos+1 < s.length()) { + if (icmp(s.charAt(pos), 'i') && icmp(s.charAt(pos+1),'d')) { + pos += 2; + if (pos < s.length()) { + switch (s.charAt(pos)) { + case '.': + { + int startPos = ++pos; + for (;pos < s.length() && (Character.toLowerCase(s.charAt(pos)) >= 'a') && (Character.toLowerCase(s.charAt(pos)) <= 'z'); pos++); + int len = pos - startPos; + if (((len == 4) && "user".equalsIgnoreCase(s.subSequence(startPos, startPos + 4).toString())) || + ((len == 5) && "group".equalsIgnoreCase(s.subSequence(startPos, startPos + 5).toString())) || + ((len == 6) && "scheme".equalsIgnoreCase(s.subSequence(startPos, startPos + 6).toString())) || + ((len == 8) && "specific".equalsIgnoreCase(s.subSequence(startPos, startPos + 8).toString())) || + ((len == 9) && "namespace".equalsIgnoreCase(s.subSequence(startPos, startPos + 9).toString()))) + { + retval = true; + id = new IdNode().setField(s.subSequence(startPos, startPos + len).toString()); + } else { + pos = startPos; + } + } + break; + case '!': + case '<': + case '>': + case '=': + case '\t': + case '\n': + case '\r': + case ' ': + { + retval = true; + id = new IdNode(); + } + break; + default: + break; + } + } + } + } + setRemaining(s.subSequence(pos, s.length())); + + return retval; + } + +} diff --git a/document/src/main/java/com/yahoo/document/select/simple/IntegerParser.java b/document/src/main/java/com/yahoo/document/select/simple/IntegerParser.java new file mode 100644 index 00000000000..52a0b0a2c4f --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/simple/IntegerParser.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.simple; + +import com.yahoo.document.select.rule.LiteralNode; + +/** + * @author balder + */ +public class IntegerParser extends Parser { + private LiteralNode value; + public LiteralNode getValue() { return value; } + + public boolean parse(CharSequence s) { + boolean retval = false; + int pos = eatWhite(s); + if (pos < s.length()) { + boolean isHex = ((s.length() - pos) > 2) && (s.charAt(pos) == '0') && (s.charAt(pos+1) == 'x'); + Long v = null; + int startPos = pos; + if (isHex) { + for(startPos = pos+2; (pos < s.length()) && (((s.charAt(pos) >= '0') && (s.charAt(pos) <= '9')) || + ((s.charAt(pos) >= 'a') && (s.charAt(pos) <= 'f')) || + ((s.charAt(pos) >= 'A') && (s.charAt(pos) <= 'F'))); pos++); + if (pos > startPos) { + v = Long.valueOf(s.subSequence(startPos, pos).toString(), 16); + retval = true; + } + + } else { + if ((s.charAt(pos) == '-') || (s.charAt(pos) == '+')) { + pos++; + } + for(;(pos < s.length()) && (s.charAt(pos) >= '0') && (s.charAt(pos) <= '9') ; pos++); + if (pos > startPos) { + v = Long.valueOf(s.subSequence(startPos, pos).toString(), 10); + retval = true; + } + } + value = new LiteralNode(v); + } + setRemaining(s.subSequence(pos, s.length())); + + return retval; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/simple/OperatorParser.java b/document/src/main/java/com/yahoo/document/select/simple/OperatorParser.java new file mode 100644 index 00000000000..58da1e0c179 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/simple/OperatorParser.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.simple; + +/** + * @author balder + */ +public class OperatorParser extends Parser { + private String operator; + public String getOperator() { return operator; } + public boolean parse(CharSequence s) { + boolean retval = false; + int pos = eatWhite(s); + + if (pos+1 < s.length()) { + retval = true; + int startPos = pos; + if (s.charAt(pos) == '=') { + pos++; + if ((s.charAt(pos) == '=') || (s.charAt(pos) == '~')) { + pos++; + } + } else if (s.charAt(pos) == '>') { + pos++; + if (s.charAt(pos) == '=') { + pos++; + } + } else if (s.charAt(pos) == '<') { + pos++; + if (s.charAt(pos) == '=') { + pos++; + } + } else if ((s.charAt(pos) == '!') && (s.charAt(pos) == '=')) { + pos += 2; + } else { + retval = false; + } + if (retval) { + operator = s.subSequence(startPos, pos).toString(); + } + } + setRemaining(s.subSequence(pos, s.length())); + + return retval; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/simple/Parser.java b/document/src/main/java/com/yahoo/document/select/simple/Parser.java new file mode 100644 index 00000000000..169f100de83 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/simple/Parser.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.simple; + +/** + * @author balder + */ +public abstract class Parser { + public abstract boolean parse(CharSequence s); + public CharSequence getRemaining() { return remaining; } + protected void setRemaining(CharSequence r) { remaining = r; } + private CharSequence remaining; + protected int eatWhite(CharSequence s) { + int pos = 0; + for (;pos < s.length() && Character.isSpaceChar(s.charAt(pos)); pos++); + return pos; + } + protected boolean icmp(char c, char l) { + return Character.toLowerCase(c) == l; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java b/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java new file mode 100644 index 00000000000..df3507e67c8 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/simple/SelectionParser.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.simple; + +import com.yahoo.document.select.rule.ComparisonNode; +import com.yahoo.document.select.rule.ExpressionNode; + +/** + * @author balder + */ +public class SelectionParser extends Parser { + private ExpressionNode node; + public ExpressionNode getNode() { return node; } + public boolean parse(CharSequence s) { + boolean retval = false; + IdSpecParser id = new IdSpecParser(); + if (id.parse(s)) { + OperatorParser op = new OperatorParser(); + if (op.parse(id.getRemaining())) { + if (id.isUserSpec()) { + IntegerParser v = new IntegerParser(); + if (v.parse(op.getRemaining())) { + node = new ComparisonNode(id.getId(), op.getOperator(), v.getValue()); + retval = true; + } + setRemaining(v.getRemaining()); + } else { + StringParser v = new StringParser(); + if (v.parse(op.getRemaining())) { + node = new ComparisonNode(id.getId(), op.getOperator(), v.getValue()); + retval = true; + } + setRemaining(v.getRemaining()); + } + } else { + setRemaining(op.getRemaining()); + } + } else { + setRemaining(id.getRemaining()); + } + + return retval; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/simple/StringParser.java b/document/src/main/java/com/yahoo/document/select/simple/StringParser.java new file mode 100644 index 00000000000..e18c2dad8e2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/simple/StringParser.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.select.simple; + +import com.yahoo.document.select.rule.LiteralNode; + +/** + * @author balder + */ +public class StringParser extends Parser { + private LiteralNode value; + public LiteralNode getValue() { return value; } + public boolean parse(CharSequence s) { + boolean retval = false; + int pos = eatWhite(s); + if (pos + 1 < s.length()) { + if (s.charAt(pos++) == '"') { + StringBuffer str = new StringBuffer(""); + for(; (pos < s.length()) && (s.charAt(pos) != '"');pos++) { + if ((pos < s.length()) && (s.charAt(pos) == '\\')) { + pos++; + } + str.append(s.charAt(pos)); + } + if (s.charAt(pos) == '"') { + pos++; + retval = true; + value = new LiteralNode(str.toString()); + } + } + + setRemaining(s.subSequence(pos, s.length())); + } + return retval; + } +} diff --git a/document/src/main/java/com/yahoo/document/select/simple/package-info.java b/document/src/main/java/com/yahoo/document/select/simple/package-info.java new file mode 100644 index 00000000000..58d4cc3428b --- /dev/null +++ b/document/src/main/java/com/yahoo/document/select/simple/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.select.simple; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/serialization/AnnotationReader.java b/document/src/main/java/com/yahoo/document/serialization/AnnotationReader.java new file mode 100644 index 00000000000..92b042249d2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/AnnotationReader.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public interface AnnotationReader { + public void read(Annotation annotation); +} diff --git a/document/src/main/java/com/yahoo/document/serialization/AnnotationWriter.java b/document/src/main/java/com/yahoo/document/serialization/AnnotationWriter.java new file mode 100644 index 00000000000..061798e91b8 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/AnnotationWriter.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public interface AnnotationWriter { + public void write(Annotation annotation); +} diff --git a/document/src/main/java/com/yahoo/document/serialization/DeserializationException.java b/document/src/main/java/com/yahoo/document/serialization/DeserializationException.java new file mode 100644 index 00000000000..a189290fb13 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DeserializationException.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +/** + * Exception which is thrown when deserialization fails. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DeserializationException extends RuntimeException { + public DeserializationException(String msg) { + super(msg); + } + + public DeserializationException(String msg, Throwable cause) { + super(msg, cause); + } + + public DeserializationException(Throwable cause) { + super(cause); + } +} diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java new file mode 100644 index 00000000000..def358e0624 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.io.GrowableByteBuffer; + +/** + * Interface for de-serializing documents. + * + * A particular instance of this class is tied to a version of the document format. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public interface DocumentDeserializer extends DocumentReader, DocumentUpdateReader, FieldReader, AnnotationReader, SpanNodeReader, SpanTreeReader { + + /** + * Returns the underlying buffer used for de-serialization. + */ + public GrowableByteBuffer getBuf(); + +} + diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java new file mode 100644 index 00000000000..cef5b024837 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.io.GrowableByteBuffer; + +/** + * Factory for creating document de-serializers tied to a document format. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DocumentDeserializerFactory { + + /** + * Creates a de-serializer for the current head document format. + * This format is an extension of the 4.2 format. + */ + public static DocumentDeserializer createHead(DocumentTypeManager manager, GrowableByteBuffer buf) { + return new VespaDocumentDeserializerHead(manager, buf); + } + + /** + * Creates a de-serializer for the document format that was created on Vespa 4.2. + */ + public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf) { + return new VespaDocumentDeserializer42(manager, buf); + } + + /** + * Creates a de-serializer for the document format that was created on Vespa 4.2. + */ + public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf, GrowableByteBuffer body) { + return new VespaDocumentDeserializer42(manager, buf, body); + } +} diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java new file mode 100644 index 00000000000..8a5e889eb8c --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentReader.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; + +/** + * This interface is used to implement custom deserialization of document updates. + * + * @author <a href="mailto:ravishar@yahoo-inc.com">Ravi Sharma</a> + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public interface DocumentReader { + + /** + * Read a document + * + * @param document - document to be read + */ + void read(Document document); + + DocumentId readDocumentId(); + DocumentType readDocumentType(); + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializer.java b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializer.java new file mode 100644 index 00000000000..6eb0fc60c3f --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializer.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.io.GrowableByteBuffer; + +/** + * Interface for serializing documents. + * + * A particular instance of this class is tied to a version of the document format. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public interface DocumentSerializer extends DocumentWriter, SpanNodeWriter, AnnotationWriter, SpanTreeWriter, DocumentUpdateWriter { + + /** + * Returns the underlying buffer used for serialization. + */ + public GrowableByteBuffer getBuf(); +} + diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java new file mode 100644 index 00000000000..50268161aad --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.io.GrowableByteBuffer; + +/** + * Factory for creating document serializers tied to a document format. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DocumentSerializerFactory { + + /** + * Creates a serializer for the current head document format. + * This format is an extension of the 4.2 format. + */ + public static DocumentSerializer createHead(GrowableByteBuffer buf) { + return new VespaDocumentSerializerHead(buf); + } + + /** + * Creates a serializer for the document format that was created on Vespa 4.2. + */ + public static DocumentSerializer create42(GrowableByteBuffer buf) { + return new VespaDocumentSerializer42(buf); + } + + /** + * Creates a serializer for the document format that was created on Vespa 4.2. + */ + public static DocumentSerializer create42(GrowableByteBuffer buf, boolean headerOnly) { + return new VespaDocumentSerializer42(buf, headerOnly); + } + + /** + * Creates a serializer for the document format that was created on Vespa 4.2. + */ + public static DocumentSerializer create42() { + return new VespaDocumentSerializer42(); + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java new file mode 100644 index 00000000000..808b5d90517 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +/** + * Class used to represent up to 4 flags used in a DocumentUpdate. + * These flags are stored as the 4 most significant bits in a 32 bit integer. + * + * Flags currently used: + * 0) create-if-non-existent. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DocumentUpdateFlags { + private byte flags; + private DocumentUpdateFlags(byte flags) { + this.flags = flags; + } + public DocumentUpdateFlags() { + this.flags = 0; + } + public boolean getCreateIfNonExistent() { + return (flags & 1) != 0; + } + public void setCreateIfNonExistent(boolean value) { + flags &= ~1; // clear flag + flags |= value ? 1 : 0; // set flag + } + public int injectInto(int value) { + return extractValue(value) | (flags << 28); + } + public static DocumentUpdateFlags extractFlags(int combined) { + return new DocumentUpdateFlags((byte)(combined >> 28)); + } + public static int extractValue(int combined) { + int mask = ~(~0 << 28); + return combined & mask; + } +}
\ No newline at end of file diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateReader.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateReader.java new file mode 100644 index 00000000000..4e31af08d00 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateReader.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.*; +import com.yahoo.document.fieldpathupdate.*; +import com.yahoo.document.update.FieldUpdate; + +/** + * This interface is used to implement custom deserialization of document updates. + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + */ +public interface DocumentUpdateReader { + + void read(DocumentUpdate update); + + void read(FieldUpdate update); + + void read(FieldPathUpdate update); + + void read(AssignFieldPathUpdate update); + + void read(AddFieldPathUpdate update); + + void read(RemoveFieldPathUpdate update); + + DocumentId readDocumentId(); + DocumentType readDocumentType(); + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java new file mode 100644 index 00000000000..725de8935f5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.ArithmeticValueUpdate; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.ClearValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.RemoveValueUpdate; + +/** + * Interface for writing document updates in custom serializers. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.27 + */ +public interface DocumentUpdateWriter { + public void write(DocumentUpdate update); + public void write(FieldUpdate update); + public void write(AddValueUpdate update, DataType superType); + public void write(MapValueUpdate update, DataType superType); + public void write(ArithmeticValueUpdate update); + public void write(AssignValueUpdate update, DataType superType); + public void write(RemoveValueUpdate update, DataType superType); + public void write(ClearValueUpdate clearValueUpdate, DataType superType); +} diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java new file mode 100644 index 00000000000..aba9c97cf67 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; + +/** + * @author <a href="mailto:ravishar@yahoo-inc.com">ravishar</a> + */ +public interface DocumentWriter extends FieldWriter { + /** + * write out a document + * + * @param document + * document to be written + */ + void write(Document document); + + void write(DocumentId id); + + void write(DocumentType type); +} diff --git a/document/src/main/java/com/yahoo/document/serialization/FieldReader.java b/document/src/main/java/com/yahoo/document/serialization/FieldReader.java new file mode 100644 index 00000000000..d66cf8bb8e1 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/FieldReader.java @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * + */ +package com.yahoo.document.serialization; + +import com.yahoo.document.Document; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.objects.Deserializer; +import com.yahoo.vespa.objects.FieldBase; + + +/** + * @author <a href="mailto:ravishar@yahoo-inc.com">ravishar</a> + * + */ +public interface FieldReader extends Deserializer { + + /** + * Read in the value of field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, Document value); + /** + * Read in the value of field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, FieldValue value); + + /** + * Read in the value of array field + * + * @param field - field description (name and data type) + * @param value - field value + */ + <T extends FieldValue> void read(FieldBase field, Array<T> value); + + /** + * Read the value of a map field + */ + <K extends FieldValue, V extends FieldValue> void read(FieldBase field, MapFieldValue<K, V> map); + + /** + * Read in the value of byte field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, ByteFieldValue value); + + /** + * Read in the value of collection field + * + * @param field - field description (name and data type) + * @param value - field value + */ + <T extends FieldValue> void read(FieldBase field, CollectionFieldValue<T> value); + + /** + * Read in the value of double field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, DoubleFieldValue value); + + /** + * Read in the value of float field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, FloatFieldValue value); + + /** + * Read in the value of integer field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, IntegerFieldValue value); + + /** + * Read in the value of long field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, LongFieldValue value); + + /** + * Read in the value of raw field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, Raw value); + + /** + * Read in the value of predicate field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, PredicateFieldValue value); + + /** + * Read in the value of string field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, StringFieldValue value); + + /** + * Read in the value of the given tensor field. + * + * @param field field description (name and data type) + * @param value tensor field value + */ + void read(FieldBase field, TensorFieldValue value); + + /** + * Read in the value of struct field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, Struct value); + + /** + * Read in the value of structured field + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, StructuredFieldValue value); + + + /** + * Read in the value of weighted set field + * + * @param field - field description (name and data type) + * @param value - field value + */ + <T extends FieldValue> void read(FieldBase field, WeightedSet<T> value); + + /** + * Read in the value of annotation reference. + * + * @param field - field description (name and data type) + * @param value - field value + */ + void read(FieldBase field, AnnotationReference value); + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java new file mode 100644 index 00000000000..4a269a704d2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java @@ -0,0 +1,193 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.Document; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.objects.FieldBase; +import com.yahoo.vespa.objects.Serializer; + +/** + * Interface for writing out com.yahoo.document.datatypes.FieldValue. + * + * @author <a href="mailto:ravishar@yahoo-inc.com">ravishar</a> + * + */ +public interface FieldWriter extends Serializer { + + /** + * Write out the value of field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, FieldValue value); + + /** + * Write out the value of field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + public void write(FieldBase field, Document value); + + /** + * Write out the value of array field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + <T extends FieldValue> void write(FieldBase field, Array<T> value); + + /** + * Write the value of a map field + */ + <K extends FieldValue, V extends FieldValue> void write(FieldBase field, + MapFieldValue<K, V> map); + + /** + * Write out the value of byte field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, ByteFieldValue value); + + /** + * Write out the value of collection field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + <T extends FieldValue> void write(FieldBase field, + CollectionFieldValue<T> value); + + /** + * Write out the value of double field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, DoubleFieldValue value); + + /** + * Write out the value of float field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, FloatFieldValue value); + + /** + * Write out the value of integer field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, IntegerFieldValue value); + + /** + * Write out the value of long field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, LongFieldValue value); + + /** + * Write out the value of raw field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, Raw value); + + /** + * Write out the value of predicate field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, PredicateFieldValue value); + + /** + * Write out the value of string field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, StringFieldValue value); + + /** + * Write out the value of the given tensor field value. + * + * @param field field description (name and data type) + * @param value tensor field value + */ + void write(FieldBase field, TensorFieldValue value); + + /** + * Write out the value of struct field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, Struct value); + + /** + * Write out the value of structured field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, StructuredFieldValue value); + + /** + * Write out the value of weighted set field + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + <T extends FieldValue> void write(FieldBase field, WeightedSet<T> value); + + /** + * Write out the value of annotation data. + * + * @param field + * field description (name and data type) + * @param value + * field value + */ + void write(FieldBase field, AnnotationReference value); + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/SerializationException.java b/document/src/main/java/com/yahoo/document/serialization/SerializationException.java new file mode 100644 index 00000000000..44dd9d2ccf8 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/SerializationException.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +/** + * Exception which is thrown when serialization fails. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SerializationException extends RuntimeException { + public SerializationException(String msg) { + super(msg); + } + + public SerializationException(String msg, Throwable cause) { + super(msg, cause); + } + + public SerializationException(Throwable cause) { + super(cause); + } +} diff --git a/document/src/main/java/com/yahoo/document/serialization/SpanNodeReader.java b/document/src/main/java/com/yahoo/document/serialization/SpanNodeReader.java new file mode 100644 index 00000000000..652758e8e38 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/SpanNodeReader.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.annotation.AlternateSpanList; +import com.yahoo.document.annotation.Span; +import com.yahoo.document.annotation.SpanList; +import com.yahoo.document.annotation.SpanNode; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public interface SpanNodeReader { + public void read(Span span); + public void read(SpanList spanList); + public void read(AlternateSpanList altSpanList); +} diff --git a/document/src/main/java/com/yahoo/document/serialization/SpanNodeWriter.java b/document/src/main/java/com/yahoo/document/serialization/SpanNodeWriter.java new file mode 100644 index 00000000000..8712c792f99 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/SpanNodeWriter.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.annotation.AlternateSpanList; +import com.yahoo.document.annotation.Span; +import com.yahoo.document.annotation.SpanList; +import com.yahoo.document.annotation.SpanNode; +import com.yahoo.vespa.objects.Serializer; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public interface SpanNodeWriter extends Serializer { + public void write(SpanNode spanNode); + public void write(Span span); + public void write(SpanList spanList); + public void write(AlternateSpanList altSpanList); +} diff --git a/document/src/main/java/com/yahoo/document/serialization/SpanTreeReader.java b/document/src/main/java/com/yahoo/document/serialization/SpanTreeReader.java new file mode 100644 index 00000000000..ca670f527b5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/SpanTreeReader.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.annotation.SpanTree; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public interface SpanTreeReader { + public void read(SpanTree tree); +} diff --git a/document/src/main/java/com/yahoo/document/serialization/SpanTreeWriter.java b/document/src/main/java/com/yahoo/document/serialization/SpanTreeWriter.java new file mode 100644 index 00000000000..629a21f149e --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/SpanTreeWriter.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.annotation.SpanTree; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public interface SpanTreeWriter { + public void write(SpanTree tree); +} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java new file mode 100644 index 00000000000..c3e51b2602a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java @@ -0,0 +1,786 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.collections.Tuple2; +import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.annotation.AlternateSpanList; +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.annotation.Span; +import com.yahoo.document.annotation.SpanList; +import com.yahoo.document.annotation.SpanNode; +import com.yahoo.document.annotation.SpanNodeParent; +import com.yahoo.document.annotation.SpanTree; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; +import com.yahoo.document.predicate.BinaryFormat; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.ArithmeticValueUpdate; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.ClearValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.RemoveValueUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8Array; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.objects.FieldBase; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.text.Utf8.calculateStringPositions; + +/** + * Class used for de-serializing documents on the Vespa 4.2 document format. + * + * @deprecated Please use {@link com.yahoo.document.serialization.VespaDocumentDeserializerHead} instead for new code. + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +@Deprecated // OK: Don't remove on Vespa 6: Mail may have documents on this format still +// When removing: Move content of this class into VespaDocumentDeserializerHead (and subclass VespaDocumentSerializerHead in that) +public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 implements DocumentDeserializer { + + private final Compressor compressor = new Compressor(); + private DocumentTypeManager manager; + GrowableByteBuffer body; + private short version; + private List<SpanNode> spanNodes; + private List<Annotation> annotations; + private int[] stringPositions; + + VespaDocumentDeserializer42(DocumentTypeManager manager, GrowableByteBuffer header, GrowableByteBuffer body, short version) { + super(header); + this.manager = manager; + this.body = body; + this.version = version; + } + + VespaDocumentDeserializer42(DocumentTypeManager manager, GrowableByteBuffer buf) { + this(manager, buf, null, Document.SERIALIZED_VERSION); + } + + VespaDocumentDeserializer42(DocumentTypeManager manager, GrowableByteBuffer buf, GrowableByteBuffer body) { + this(manager, buf, body, Document.SERIALIZED_VERSION); + } + + final public DocumentTypeManager getDocumentTypeManager() { return manager; } + + public void read(Document document) { + read(null, document); + } + public void read(FieldBase field, Document doc) { + + // Verify that we have correct version + version = getShort(null); + if (version < 6 || version > Document.SERIALIZED_VERSION) { + throw new DeserializationException( + "Unknown version " + version + ", expected " + Document.SERIALIZED_VERSION + "."); + } + + int dataLength = 0; + int dataPos = 0; + + if (version < 7) { + getInt2_4_8Bytes(null); // Total document size.. Ignore + } else { + dataLength = getInt(null); + dataPos = position(); + } + + doc.setId(readDocumentId()); + + Byte content = getByte(null); + + doc.setDataType(readDocumentType()); + + if ((content & 0x2) != 0) { + doc.getHeader().deserialize(new Field("header"),this); + } + if ((content & 0x4) != 0) { + doc.getBody().deserialize(new Field("body"),this); + } else if (body != null) { + GrowableByteBuffer header = getBuf(); + setBuf(body); + body = null; + doc.getBody().deserialize(new Field("body"), this); + body = getBuf(); + setBuf(header); + } + + if (version < 8) { + int crcVal = getInt(null); + } + + if (version > 6) { + if (dataLength != (position() - dataPos)) { + throw new DeserializationException("Length mismatch"); + } + } + } + public void read(FieldBase field, FieldValue value) { + throw new IllegalArgumentException("read not implemented yet."); + } + + public <T extends FieldValue> void read(FieldBase field, Array<T> array) { + int numElements = getNumCollectionElems(); + ArrayList<T> list = new ArrayList<T>(numElements); + ArrayDataType type = array.getDataType(); + for (int i = 0; i < numElements; i++) { + if (version < 7) { + getInt(null); // We don't need size for anything + } + FieldValue fv = type.getNestedType().createFieldValue(); + fv.deserialize(null, this); + list.add((T) fv); + } + array.clear(); + array.addAll(list); + } + + public <K extends FieldValue, V extends FieldValue> void read(FieldBase field, MapFieldValue<K, V> map) { + int numElements = getNumCollectionElems(); + Map<K,V> hash = new HashMap<>(); + MapDataType type = map.getDataType(); + for (int i = 0; i < numElements; i++) { + if (version < 7) { + getInt(null); // We don't need size for anything + } + K key = (K) type.getKeyType().createFieldValue(); + V val = (V) type.getValueType().createFieldValue(); + key.deserialize(null, this); + val.deserialize(null, this); + hash.put(key, val); + } + map.clear(); + map.putAll(hash); + } + + private int getNumCollectionElems() { + int numElements; + if (version < 7) { + getInt(null); // We already know the nested type, so ignore that.. + numElements = getInt(null); + } else { + numElements = getInt1_2_4Bytes(null); + } + if (numElements < 0) { + throw new DeserializationException("Bad number of array/map elements, " + numElements); + } + return numElements; + } + + public <T extends FieldValue> void read(FieldBase field, CollectionFieldValue<T> value) { + throw new IllegalArgumentException("read not implemented yet."); + } + public void read(FieldBase field, ByteFieldValue value) { value.assign(getByte(null)); } + public void read(FieldBase field, DoubleFieldValue value) { value.assign(getDouble(null)); } + public void read(FieldBase field, FloatFieldValue value) { value.assign(getFloat(null)); } + public void read(FieldBase field, IntegerFieldValue value) { value.assign(getInt(null)); } + public void read(FieldBase field, LongFieldValue value) { value.assign(getLong(null)); } + + public void read(FieldBase field, Raw value) { + int rawsize = getInt(null); + byte[] rawBytes = getBytes(null, rawsize); + value.assign(rawBytes); + } + + @Override + public void read(FieldBase field, PredicateFieldValue value) { + int len = getInt(null); + byte[] buf = getBytes(null, len); + value.assign(BinaryFormat.decode(buf)); + } + + public void read(FieldBase field, StringFieldValue value) { + byte coding = getByte(null); + + int length = getInt1_4Bytes(null); + + //OK, it seems that this length includes null termination. + //NOTE: the following four lines are basically parseNullTerminatedString() inlined, + //but we need to use the UTF-8 buffer below, so not using that method... + byte[] stringArray = new byte[length - 1]; + buf.get(stringArray); + buf.get(); //move past 0-termination + value.setUnChecked(Utf8.toString(stringArray)); + + if ((coding & 64) == 64) { + //we have a span tree! + try { + //we don't support serialization of nested span trees, so this is safe: + stringPositions = calculateStringPositions(stringArray); + //total length: + int size = buf.getInt(); + int startPos = buf.position(); + + int numSpanTrees = buf.getInt1_2_4Bytes(); + + for (int i = 0; i < numSpanTrees; i++) { + SpanTree tree = new SpanTree(); + StringFieldValue treeName = new StringFieldValue(); + treeName.deserialize(this); + tree.setName(treeName.getString()); + value.setSpanTree(tree); + readSpanTree(tree, false); + } + + buf.position(startPos + size); + } finally { + stringPositions = null; + } + } + } + + @Override + public void read(FieldBase field, TensorFieldValue value) { + int encodedTensorLength = buf.getInt1_4Bytes(); + if (encodedTensorLength > 0) { + byte[] encodedTensor = getBytes(null, encodedTensorLength); + value.assign(TypedBinaryFormat.decode(encodedTensor)); + } else { + value.clear(); + } + } + + public void read(FieldBase fieldDef, Struct s) { + s.setVersion(version); + int startPos = position(); + + if (version < 6) { + throw new DeserializationException("Illegal document serialization version " + version); + } + + int dataSize; + if (version < 7) { + long rSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (rSize > Integer.MAX_VALUE) { + throw new DeserializationException("Raw size of data block is too large."); + } + dataSize = (int)rSize; + } else { + dataSize = getInt(null); + } + + byte comprCode = getByte(null); + CompressionType compression = CompressionType.valueOf(comprCode); + + int uncompressedSize = 0; + if (compression != CompressionType.NONE && + compression != CompressionType.INCOMPRESSIBLE) + { + // uncompressedsize (full size of FIELDS only, after decompression) + long pSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (pSize > Integer.MAX_VALUE) { + throw new DeserializationException("Uncompressed size of data block is too large."); + } + uncompressedSize = (int) pSize; + } + + int numberOfFields = getInt1_4Bytes(null); + + List<Tuple2<Integer, Long>> fieldIdsAndLengths = new ArrayList<>(numberOfFields); + for (int i=0; i<numberOfFields; ++i) { + // id, length (length only used for unknown fields + fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null))); + } + + //save a reference to the big buffer we're reading from: + GrowableByteBuffer bigBuf = buf; + + if (version < 7) { + // In V6 and earlier, the length included the header. + int headerSize = position() - startPos; + dataSize -= headerSize; + } + byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize)); + + // set position in original buffer to after data + position(position() + dataSize); + + // for a while: deserialize from this buffer instead: + buf = GrowableByteBuffer.wrap(destination); + + s.clear(); + StructDataType type = s.getDataType(); + for (int i=0; i<numberOfFields; ++i) { + Field structField = type.getField(fieldIdsAndLengths.get(i).first, version); + if (structField == null) { + //ignoring unknown field: + position(position() + fieldIdsAndLengths.get(i).second.intValue()); + } else { + int posBefore = position(); + FieldValue value = structField.getDataType().createFieldValue(); + value.deserialize(structField, this); + s.setFieldValue(structField, value); + //jump to beginning of next field: + position(posBefore + fieldIdsAndLengths.get(i).second.intValue()); + } + } + + // restore the original buffer + buf = bigBuf; + } + + public void read(FieldBase field, StructuredFieldValue value) { + throw new IllegalArgumentException("read not implemented yet."); + } + public <T extends FieldValue> void read(FieldBase field, WeightedSet<T> ws) { + WeightedSetDataType type = ws.getDataType(); + getInt(null); // Have no need for type + + int numElements = getInt(null); + if (numElements < 0) { + throw new DeserializationException("Bad number of weighted set elements, " + numElements); + } + + ws.clearAndReserve(numElements * 2); // Avoid resizing + for (int i = 0; i < numElements; i++) { + int size = getInt(null); + FieldValue value = type.getNestedType().createFieldValue(); + value.deserialize(null, this); + IntegerFieldValue weight = new IntegerFieldValue(getInt(null)); + ws.putUnChecked((T) value, weight); + } + + } + + public void read(FieldBase field, AnnotationReference value) { + int seqId = buf.getInt1_2_4Bytes(); + try { + Annotation a = annotations.get(seqId); + value.setReferenceNoCompatibilityCheck(a); + } catch (IndexOutOfBoundsException iiobe) { + throw new SerializationException("Could not serialize AnnotationReference value, reference not found.", iiobe); + } + } + + private Utf8String deserializeAttributeString() throws DeserializationException { + int length = getByte(null); + return new Utf8String(parseNullTerminatedString(length)); + } + + private Utf8Array parseNullTerminatedString() { return parseNullTerminatedString(getBuf().getByteBuffer()); } + private Utf8Array parseNullTerminatedString(int lengthExcludingNull) { return parseNullTerminatedString(getBuf().getByteBuffer(), lengthExcludingNull); } + + static Utf8Array parseNullTerminatedString(ByteBuffer buf, int lengthExcludingNull) throws DeserializationException { + Utf8Array utf8 = new Utf8Array(buf, lengthExcludingNull); + buf.get(); //move past 0-termination + return utf8; + } + + static Utf8Array parseNullTerminatedString(ByteBuffer buf) throws DeserializationException { + //search for 0-byte + int end = getFirstNullByte(buf); + + if (end == -1) { + throw new DeserializationException("Could not locate terminating 0-byte for string"); + } + + return parseNullTerminatedString(buf, end - buf.position()); + } + + private static int getFirstNullByte(ByteBuffer buf) { + int end = -1; + int start = buf.position(); + + while (true) { + try { + byte dataByte = buf.get(); + if (dataByte == (byte) 0) { + end = buf.position() - 1; + break; + } + } catch (Exception e) { + break; + } + } + + buf.position(start); + return end; + } + + public void read(DocumentUpdate update) { + short serializationVersion = getShort(null); + + update.setId(new DocumentId(this)); + + byte contents = getByte(null); + + if ((contents & 0x1) == 0) { + throw new DeserializationException("Cannot deserialize DocumentUpdate without doctype"); + } + + update.setDocumentType(readDocumentType()); + + int size = getInt(null); + + for (int i = 0; i < size; i++) { + // TODO: Should use checked method, but doesn't work according to test now. + update.addFieldUpdateNoCheck(new FieldUpdate(this, update.getDocumentType(), serializationVersion)); + } + } + + public void read(FieldPathUpdate update) { + String fieldPath = getString(null); + String whereClause = getString(null); + update.setFieldPath(fieldPath); + + try { + update.setWhereClause(whereClause); + } catch (ParseException e) { + throw new DeserializationException(e); + } + } + + public void read(AssignFieldPathUpdate update) { + byte flags = getByte(null); + update.setRemoveIfZero((flags & AssignFieldPathUpdate.REMOVE_IF_ZERO) != 0); + update.setCreateMissingPath((flags & AssignFieldPathUpdate.CREATE_MISSING_PATH) != 0); + if ((flags & AssignFieldPathUpdate.ARITHMETIC_EXPRESSION) != 0) { + update.setExpression(getString(null)); + } else { + DataType dt = update.getFieldPath().getResultingDataType(); + FieldValue fv = dt.createFieldValue(); + fv.deserialize(this); + update.setNewValue(fv); + } + } + + public void read(RemoveFieldPathUpdate update) { + + } + + public void read(AddFieldPathUpdate update) { + DataType dt = update.getFieldPath().getResultingDataType(); + FieldValue fv = dt.createFieldValue(); + dt.createFieldValue(); + fv.deserialize(this); + + if (!(fv instanceof Array)) { + throw new DeserializationException("Add only applicable to array types"); + } + update.setNewValues((Array)fv); + } + + public ValueUpdate getValueUpdate(DataType superType, DataType subType) { + int vuTypeId = getInt(null); + + ValueUpdate.ValueUpdateClassID op = ValueUpdate.ValueUpdateClassID.getID(vuTypeId); + if (op == null) { + throw new IllegalArgumentException("Read type "+vuTypeId+" of bytebuffer, but this is not a legal value update type."); + } + + switch (op) { + case ADD: + { + FieldValue fval = subType.createFieldValue(); + fval.deserialize(this); + int weight = getInt(null); + return new AddValueUpdate(fval, weight); + } + case ARITHMETIC: + int opId = getInt(null); + ArithmeticValueUpdate.Operator operator = ArithmeticValueUpdate.Operator.getID(opId); + double operand = getDouble(null); + return new ArithmeticValueUpdate(operator, operand); + case ASSIGN: + { + byte contents = getByte(null); + FieldValue fval = null; + if (contents == (byte) 1) { + fval = superType.createFieldValue(); + fval.deserialize(this); + } + return new AssignValueUpdate(fval); + } + case CLEAR: + return new ClearValueUpdate(); + case MAP: + if (superType instanceof ArrayDataType) { + CollectionDataType type = (CollectionDataType) superType; + IntegerFieldValue index = new IntegerFieldValue(); + index.deserialize(this); + ValueUpdate update = getValueUpdate(type.getNestedType(), null); + return new MapValueUpdate(index, update); + } else if (superType instanceof WeightedSetDataType) { + CollectionDataType type = (CollectionDataType) superType; + FieldValue fval = type.getNestedType().createFieldValue(); + fval.deserialize(this); + ValueUpdate update = getValueUpdate(DataType.INT, null); + return new MapValueUpdate(fval, update); + } else { + throw new DeserializationException("MapValueUpdate only works for arrays and weighted sets"); + } + case REMOVE: + FieldValue fval = ((CollectionDataType) superType).getNestedType().createFieldValue(); + fval.deserialize(this); + return new RemoveValueUpdate(fval); + default: + throw new DeserializationException( + "Could not deserialize ValueUpdate, unknown valueUpdateClassID type " + vuTypeId); + } + } + + public void read(FieldUpdate fieldUpdate) { + int fieldId = getInt(null); + Field field = fieldUpdate.getDocumentType().getField(fieldId, fieldUpdate.getSerializationVersion()); + if (field == null) { + throw new DeserializationException( + "Cannot deserialize FieldUpdate, field fieldId " + fieldId + " not found in " + fieldUpdate.getDocumentType()); + } + + fieldUpdate.setField(field); + int size = getInt(null); + + for (int i = 0; i < size; i++) { + if (field.getDataType() instanceof CollectionDataType) { + CollectionDataType collType = (CollectionDataType) field.getDataType(); + fieldUpdate.addValueUpdate(getValueUpdate(collType, collType.getNestedType())); + } else { + fieldUpdate.addValueUpdate(getValueUpdate(field.getDataType(), null)); + } + } + } + + public DocumentId readDocumentId() { + Utf8String uri = new Utf8String(parseNullTerminatedString(getBuf().getByteBuffer())); + return new DocumentId(uri.toString()); + } + + public DocumentType readDocumentType() { + Utf8Array docTypeName = parseNullTerminatedString(); + int ignored = getShort(null); // used to hold the version + + DocumentType docType = manager.getDocumentType(new DataTypeName(docTypeName)); + if (docType == null) { + throw new DeserializationException( + "No known document type with name " + new Utf8String(docTypeName).toString()); + } + return docType; + } + + private SpanNode readSpanNode() { + byte type = buf.get(); + buf.position(buf.position() - 1); + + SpanNode retval; + if ((type & Span.ID) == Span.ID) { + retval = new Span(); + if (spanNodes != null) { + spanNodes.add(retval); + } + read((Span) retval); + } else if ((type & SpanList.ID) == SpanList.ID) { + retval = new SpanList(); + if (spanNodes != null) { + spanNodes.add(retval); + } + read((SpanList) retval); + } else if ((type & AlternateSpanList.ID) == AlternateSpanList.ID) { + retval = new AlternateSpanList(); + if (spanNodes != null) { + spanNodes.add(retval); + } + read((AlternateSpanList) retval); + } else { + throw new DeserializationException("Cannot read SpanNode of type " + type); + } + return retval; + } + + private void readSpanTree(SpanTree tree, boolean readName) { + //we don't support serialization of nested span trees: + if (spanNodes != null || annotations != null) { + throw new SerializationException("Deserialization of nested SpanTrees is not supported."); + } + + //we're going to write a new SpanTree, create a new Map for nodes: + spanNodes = new ArrayList<SpanNode>(); + annotations = new ArrayList<Annotation>(); + + try { + if (readName) { + StringFieldValue treeName = new StringFieldValue(); + treeName.deserialize(this); + tree.setName(treeName.getString()); + } + + SpanNode root = readSpanNode(); + tree.setRoot(root); + + int numAnnotations = buf.getInt1_2_4Bytes(); + + for (int i = 0; i < numAnnotations; i++) { + Annotation a = new Annotation(); + annotations.add(a); + } + for (int i = 0; i < numAnnotations; i++) { + read(annotations.get(i)); + } + for (Annotation a : annotations) { + tree.annotate(a); + } + + for (SpanNode node: spanNodes) { + if (node instanceof Span) { + correctIndexes((Span) node); + } + } + } finally { + //we're done, let's set this to null to save memory and prevent madness: + spanNodes = null; + annotations = null; + } + } + + public void read(SpanTree tree) { + readSpanTree(tree, true); + } + + public void read(Annotation annotation) { + int annotationTypeId = buf.getInt(); + AnnotationType type = manager.getAnnotationTypeRegistry().getType(annotationTypeId); + + if (type == null) { + throw new DeserializationException("Cannot deserialize annotation of type " + annotationTypeId + " (unknown type)"); + } + + annotation.setType(type); + + byte features = buf.get(); + int length = buf.getInt1_2_4Bytes(); + + if ((features & (byte) 1) == (byte) 1) { + //we have a span node + int spanNodeId = buf.getInt1_2_4Bytes(); + try { + SpanNode node = spanNodes.get(spanNodeId); + annotation.setSpanNode(node); + } catch (IndexOutOfBoundsException ioobe) { + throw new DeserializationException("Could not deserialize annotation, associated span node not found ", ioobe); + } + } + if ((features & (byte) 2) == (byte) 2) { + //we have a value: + int dataTypeId = buf.getInt(); + + //if this data type ID the same as the one in our config? + if (dataTypeId != type.getDataType().getId()) { + //not the same, but we will handle it gracefully, and just skip past the data: + buf.position(buf.position() + length - 4); + } else { + FieldValue value = type.getDataType().createFieldValue(); + value.deserialize(this); + annotation.setFieldValue(value); + } + } + } + + public void read(Span span) { + byte type = buf.get(); + if ((type & Span.ID) != Span.ID) { + throw new DeserializationException("Cannot deserialize Span with type " + type); + } + span.setFrom(buf.getInt1_2_4Bytes()); + span.setLength(buf.getInt1_2_4Bytes()); + } + + private void correctIndexes(Span span) { + if (stringPositions == null) { + throw new DeserializationException("Cannot deserialize Span, no access to parent StringFieldValue."); + } + int fromIndex = stringPositions[span.getFrom()]; + int toIndex = stringPositions[span.getTo()]; + int length = toIndex - fromIndex; + + span.setFrom(fromIndex); + span.setLength(length); + } + + public void read(SpanList spanList) { + byte type = buf.get(); + if ((type & SpanList.ID) != SpanList.ID) { + throw new DeserializationException("Cannot deserialize SpanList with type " + type); + } + List<SpanNode> nodes = readSpanList(spanList); + for (SpanNode node : nodes) { + spanList.add(node); + } + } + + public void read(AlternateSpanList altSpanList) { + byte type = buf.get(); + if ((type & AlternateSpanList.ID) != AlternateSpanList.ID) { + throw new DeserializationException("Cannot deserialize AlternateSpanList with type " + type); + } + int numSubTrees = buf.getInt1_2_4Bytes(); + + for (int i = 0; i < numSubTrees; i++) { + double prob = buf.getDouble(); + List<SpanNode> list = readSpanList(altSpanList); + + if (i == 0) { + for (SpanNode node : list) { + altSpanList.add(node); + } + altSpanList.setProbability(0, prob); + } else { + altSpanList.addChildren(i, list, prob); + } + } + } + + private List<SpanNode> readSpanList(SpanNodeParent parent) { + int size = buf.getInt1_2_4Bytes(); + List<SpanNode> spanList = new ArrayList<SpanNode>(); + for (int i = 0; i < size; i++) { + spanList.add(readSpanNode()); + } + return spanList; + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java new file mode 100644 index 00000000000..927075a25b5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.io.GrowableByteBuffer; + +/** + * Class used for de-serializing documents on the current head document format. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer42 { + + public VespaDocumentDeserializerHead(DocumentTypeManager manager, GrowableByteBuffer buffer) { + super(manager, buffer); + } + + @Override + public void read(DocumentUpdate update) { + update.setId(new DocumentId(this)); + update.setDocumentType(readDocumentType()); + + int size = getInt(null); + + for (int i = 0; i < size; i++) { + // TODO: Should use checked method, but doesn't work according to test now. + update.addFieldUpdateNoCheck(new FieldUpdate(this, update.getDocumentType(), 8)); + } + + try { + int sizeAndFlags = getInt(null); + update.setCreateIfNonExistent(DocumentUpdateFlags.extractFlags(sizeAndFlags).getCreateIfNonExistent()); + size = DocumentUpdateFlags.extractValue(sizeAndFlags); + + for (int i = 0; i < size; i++) { + int type = getByte(null); + update.addFieldPathUpdate(FieldPathUpdate.create(FieldPathUpdate.Type.valueOf(type), + update.getDocumentType(), this)); + } + } catch (ParseException e) { + throw new DeserializationException(e); + } + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java new file mode 100644 index 00000000000..64aea73d000 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java @@ -0,0 +1,644 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.compress.Compressor; +import com.yahoo.document.*; +import com.yahoo.document.annotation.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.predicate.BinaryFormat; +import com.yahoo.document.update.*; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import com.yahoo.vespa.objects.BufferSerializer; +import com.yahoo.vespa.objects.FieldBase; + +import java.nio.ByteBuffer; +import java.util.*; +import java.util.logging.Logger; + +import static com.yahoo.text.Utf8.calculateBytePositions; + +/** + * Class used for serializing documents on the Vespa 4.2 document format. + * + * @deprecated Please use {@link com.yahoo.document.serialization.VespaDocumentSerializerHead} instead for new code. + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +@Deprecated // OK: Don't remove on Vespa 6: Mail may have documents on this format still +// When removing: Move content into VespaDocumentSerializerHead +public class VespaDocumentSerializer42 extends BufferSerializer implements DocumentSerializer { + + private final Compressor compressor = new Compressor(); + private final static Logger log = Logger.getLogger(VespaDocumentSerializer42.class.getName()); + private boolean headerOnly; + private int spanNodeCounter = -1; + private int[] bytePositions; + + VespaDocumentSerializer42(GrowableByteBuffer buf) { + super(buf); + } + + VespaDocumentSerializer42(ByteBuffer buf) { + super(buf); + } + + VespaDocumentSerializer42(byte[] buf) { + super(buf); + } + + VespaDocumentSerializer42() { + super(); + } + + VespaDocumentSerializer42(GrowableByteBuffer buf, boolean headerOnly) { + this(buf); + this.headerOnly = headerOnly; + } + + public void setHeaderOnly(boolean headerOnly) { + this.headerOnly = headerOnly; + } + + public void write(Document doc) { + write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc); + } + + public void write(FieldBase field, Document doc) { + //save the starting position in the buffer + int startPos = buf.position(); + + buf.putShort(Document.SERIALIZED_VERSION); + + // Temporary length, fill in after serialization is done. + buf.putInt(0); + + doc.getId().serialize(this); + + byte contents = 0x01; // Indicating we have document type which we always have + if (doc.getHeader().getFieldCount() > 0) { + contents |= 0x2; // Indicate we have header + } + if (!headerOnly && doc.getBody().getFieldCount() > 0) { + contents |= 0x4; // Indicate we have a body + } + buf.put(contents); + + doc.getDataType().serialize(this); + + if (doc.getHeader().getFieldCount() > 0) { + doc.getHeader().serialize(doc.getDataType().getField("header"), this); + } + + if (!headerOnly && doc.getBody().getFieldCount() > 0) { + doc.getBody().serialize(doc.getDataType().getField("body"), this); + } + + int finalPos = buf.position(); + + buf.position(startPos + 2); + buf.putInt(finalPos - startPos - 2 - 4); // Don't include the length itself or the version + buf.position(finalPos); + + } + + /** + * Write out the value of field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, FieldValue value) { + throw new IllegalArgumentException("Not Implemented"); + } + + /** + * Write out the value of array field + * + * @param field - field description (name and data type) + * @param array - field value + */ + public <T extends FieldValue> void write(FieldBase field, Array<T> array) { + buf.putInt1_2_4Bytes(array.size()); + + List<T> lst = array.getValues(); + for (FieldValue value : lst) { + value.serialize(this); + } + + } + + public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) { + buf.putInt1_2_4Bytes(map.size()); + for (Map.Entry<K, V> e : map.entrySet()) { + e.getKey().serialize(this); + e.getValue().serialize(this); + } + } + + /** + * Write out the value of byte field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, ByteFieldValue value) { + buf.put(value.getByte()); + } + + /** + * Write out the value of collection field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { + throw new IllegalArgumentException("Not Implemented"); + } + + /** + * Write out the value of double field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, DoubleFieldValue value) { + buf.putDouble(value.getDouble()); + } + + /** + * Write out the value of float field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, FloatFieldValue value) { + buf.putFloat(value.getFloat()); + } + + /** + * Write out the value of integer field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, IntegerFieldValue value) { + buf.putInt(value.getInteger()); + } + + /** + * Write out the value of long field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, LongFieldValue value) { + buf.putLong(value.getLong()); + } + + /** + * Write out the value of raw field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, Raw value) { + ByteBuffer rawBuf = value.getByteBuffer(); + int origPos = rawBuf.position(); + buf.putInt(rawBuf.remaining()); + buf.put(rawBuf); + rawBuf.position(origPos); + + } + + @Override + public void write(FieldBase field, PredicateFieldValue value) { + byte[] buf = BinaryFormat.encode(value.getPredicate()); + this.buf.putInt(buf.length); + this.buf.put(buf); + } + + /** + * Write out the value of string field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, StringFieldValue value) { + byte[] stringBytes = createUTF8CharArray(value.getString()); + + byte coding = 0; + //Use bit 6 of "coding" to say whether span tree is available or not + if (!value.getSpanTrees().isEmpty()) { + coding |= 64; + } + buf.put(coding); + buf.putInt1_4Bytes(stringBytes.length + 1); + + buf.put(stringBytes); + buf.put(((byte) 0)); + + Map<String, SpanTree> trees = value.getSpanTreeMap(); + if ((trees != null) && !trees.isEmpty()) { + try { + //we don't support serialization of nested span trees, so this is safe: + bytePositions = calculateBytePositions(value.getString()); + //total length. record position and go back here if necessary: + int posBeforeSize = buf.position(); + buf.putInt(0); + buf.putInt1_2_4Bytes(trees.size()); + + for (SpanTree tree : trees.values()) { + try { + write(tree); + } catch (SerializationException e) { + throw e; + } catch (RuntimeException e) { + throw new SerializationException("Exception thrown while serializing span tree '" + + tree.getName() + "'; string='" + value.getString() + "'", e); + } + } + int endPos = buf.position(); + buf.position(posBeforeSize); + buf.putInt(endPos - posBeforeSize - 4); //length shall exclude itself + buf.position(endPos); + } finally { + bytePositions = null; + } + } + } + + @Override + public void write(FieldBase field, TensorFieldValue value) { + if (value.getTensor().isPresent()) { + byte[] encodedTensor = TypedBinaryFormat.encode(value.getTensor().get()); + buf.putInt1_4Bytes(encodedTensor.length); + buf.put(encodedTensor); + } else { + buf.putInt1_4Bytes(0); + } + } + + /** + * Write out the value of struct field + * + * @param field - field description (name and data type) + * @param s - field value + */ + public void write(FieldBase field, Struct s) { + // Serialize all parts first.. As we need to know length before starting + // Serialize all the fields. + + //keep the buffer we're serializing everything into: + GrowableByteBuffer bigBuffer = buf; + + //create a new buffer and serialize into that for a while: + GrowableByteBuffer buffer = new GrowableByteBuffer(4096, 2.0f); + buf = buffer; + + List<Integer> fieldIds = new LinkedList<>(); + List<java.lang.Integer> fieldLengths = new LinkedList<>(); + + for (Map.Entry<Field, FieldValue> value : s.getFields()) { + + int startPos = buffer.position(); + value.getValue().serialize(value.getKey(), this); + + fieldLengths.add(buffer.position() - startPos); + fieldIds.add(value.getKey().getId(s.getVersion())); + } + + // Switch buffers again: + buffer.flip(); + buf = bigBuffer; + + int uncompressedSize = buffer.remaining(); + Compressor.Compression compression = + s.getDataType().getCompressor().compress(buffer.getByteBuffer().array(), buffer.remaining()); + + // Actual serialization starts here. + int lenPos = buf.position(); + putInt(null, 0); // Move back to this after compression is done. + buf.put(compression.type().getCode()); + + if (compression.data() != null && compression.type().isCompressed()) { + buf.putInt2_4_8Bytes(uncompressedSize); + } + + buf.putInt1_4Bytes(s.getFieldCount()); + + for (int i = 0; i < s.getFieldCount(); ++i) { + putInt1_4Bytes(null, fieldIds.get(i)); + putInt2_4_8Bytes(null, fieldLengths.get(i)); + } + + int pos = buf.position(); + if (compression.data() != null) { + put(null, compression.data()); + } else { + put(null, buffer.getByteBuffer()); + } + int dataLength = buf.position() - pos; + + int posNow = buf.position(); + buf.position(lenPos); + putInt(null, dataLength); + buf.position(posNow); + } + + /** + * Write out the value of structured field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, StructuredFieldValue value) { + throw new IllegalArgumentException("Not Implemented"); + } + + /** + * Write out the value of weighted set field + * + * @param field - field description (name and data type) + * @param ws - field value + */ + public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> ws) { + WeightedSetDataType type = ws.getDataType(); + putInt(null, type.getNestedType().getId()); + putInt(null, ws.size()); + + Iterator<T> it = ws.fieldValueIterator(); + while (it.hasNext()) { + FieldValue key = it.next(); + java.lang.Integer value = ws.get(key); + int sizePos = buf.position(); + putInt(null, 0); + int startPos = buf.position(); + key.serialize(this); + putInt(null, value); + int finalPos = buf.position(); + int size = finalPos - startPos; + buf.position(sizePos); + putInt(null, size); + buf.position(finalPos); + } + + } + + public void write(FieldBase field, AnnotationReference value) { + int annotationId = value.getReference().getScratchId(); + if (annotationId >= 0) { + buf.putInt1_2_4Bytes(annotationId); + } else { + throw new SerializationException("Could not serialize AnnotationReference value, reference not found (" + value + ")"); + } + } + + public void write(DocumentId id) { + put(null, id.getScheme().toUtf8().getBytes()); + putByte(null, (byte) 0); + } + + public void write(DocumentType type) { + byte[] docType = createUTF8CharArray(type.getName()); + put(null, docType); + putByte(null, ((byte) 0)); + putShort(null, (short) 0); // Used to hold the version. Is now always 0. + } + + + private static void serializeAttributeString(GrowableByteBuffer data, String input) { + byte[] inputBytes = createUTF8CharArray(input); + data.put((byte) (inputBytes.length)); + data.put(inputBytes); + data.put((byte) 0); + } + + public void write(Annotation annotation) { + buf.putInt(annotation.getType().getId()); //name hash + + byte features = 0; + if (annotation.isSpanNodeValid()) { + features |= ((byte) 1); + } + if (annotation.hasFieldValue()) { + features |= ((byte) 2); + } + buf.put(features); + + int posBeforeSize = buf.position(); + buf.putInt1_2_4BytesAs4(0); + + //write ID of span node: + if (annotation.isSpanNodeValid()) { + int spanNodeId = annotation.getSpanNode().getScratchId(); + if (spanNodeId >= 0) { + buf.putInt1_2_4Bytes(spanNodeId); + } else { + throw new SerializationException("Could not serialize annotation, associated SpanNode not found (" + annotation + ")"); + } + } + + //write annotation value: + if (annotation.hasFieldValue()) { + buf.putInt(annotation.getType().getDataType().getId()); + annotation.getFieldValue().serialize(this); + } + + int end = buf.position(); + buf.position(posBeforeSize); + buf.putInt1_2_4BytesAs4(end - posBeforeSize - 4); + buf.position(end); + } + + public void write(SpanTree tree) { + //we don't support serialization of nested span trees: + if (spanNodeCounter >= 0) { + throw new SerializationException("Serialization of nested SpanTrees is not supported."); + } + + //we're going to write a new SpanTree, create a new Map for nodes: + spanNodeCounter = 0; + + //make sure tree is consistent before continuing: + tree.cleanup(); + + try { + new StringFieldValue(tree.getName()).serialize(this); + + write(tree.getRoot()); + { + //add all annotations to temporary list and sort it, to get predictable serialization + List<Annotation> tmpAnnotationList = new ArrayList<Annotation>(tree.numAnnotations()); + for (Annotation annotation : tree) { + tmpAnnotationList.add(annotation); + } + Collections.sort(tmpAnnotationList); + + int annotationCounter = 0; + //add all annotations to map here, in case of back-references: + for (Annotation annotation : tmpAnnotationList) { + annotation.setScratchId(annotationCounter++); + } + + buf.putInt1_2_4Bytes(tmpAnnotationList.size()); + for (Annotation annotation : tmpAnnotationList) { + write(annotation); + } + } + } finally { + //we're done, let's set these to null to save memory and prevent madness: + spanNodeCounter = -1; + } + } + + public void write(SpanNode spanNode) { + if (spanNodeCounter >= 0) { + spanNode.setScratchId(spanNodeCounter++); + } + if (spanNode instanceof Span) { + write((Span) spanNode); + } else if (spanNode instanceof AlternateSpanList) { + write((AlternateSpanList) spanNode); + } else if (spanNode instanceof SpanList) { + write((SpanList) spanNode); + } else { + throw new IllegalStateException("BUG!! Unable to serialize " + spanNode); + } + } + + public void write(Span span) { + buf.put(Span.ID); + + if (bytePositions != null) { + int byteFrom = bytePositions[span.getFrom()]; + int byteLength = bytePositions[span.getFrom() + span.getLength()] - byteFrom; + + buf.putInt1_2_4Bytes(byteFrom); + buf.putInt1_2_4Bytes(byteLength); + } else { + throw new SerializationException("Cannot serialize Span " + span + ", no access to parent StringFieldValue."); + } + } + + public void write(SpanList spanList) { + buf.put(SpanList.ID); + buf.putInt1_2_4Bytes(spanList.numChildren()); + Iterator<SpanNode> children = spanList.childIterator(); + while (children.hasNext()) { + write(children.next()); + } + } + + public void write(AlternateSpanList altSpanList) { + buf.put(AlternateSpanList.ID); + buf.putInt1_2_4Bytes(altSpanList.getNumSubTrees()); + for (int i = 0; i < altSpanList.getNumSubTrees(); i++) { + buf.putDouble(altSpanList.getProbability(i)); + buf.putInt1_2_4Bytes(altSpanList.numChildren(i)); + Iterator<SpanNode> children = altSpanList.childIterator(i); + while (children.hasNext()) { + write(children.next()); + } + } + } + + @Override + public void write(DocumentUpdate update) { + putShort(null, Document.SERIALIZED_VERSION); + update.getId().serialize(this); + + byte contents = 0x1; // Legacy to say we have document type + putByte(null, contents); + update.getDocumentType().serialize(this); + + putInt(null, update.getFieldUpdates().size()); + + for (FieldUpdate up : update.getFieldUpdates()) { + up.serialize(this); + } + } + + @Override + public void write(FieldUpdate update) { + putInt(null, update.getField().getId(Document.SERIALIZED_VERSION)); + putInt(null, update.getValueUpdates().size()); + for (ValueUpdate vupd : update.getValueUpdates()) { + putInt(null, vupd.getValueUpdateClassID().id); + vupd.serialize(this, update.getField().getDataType()); + } + } + + @Override + public void write(AddValueUpdate update, DataType superType) { + writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); + putInt(null, update.getWeight()); + } + + @Override + public void write(MapValueUpdate update, DataType superType) { + if (superType instanceof ArrayDataType) { + CollectionDataType type = (CollectionDataType) superType; + IntegerFieldValue index = (IntegerFieldValue) update.getValue(); + index.serialize(this); + putInt(null, update.getUpdate().getValueUpdateClassID().id); + update.getUpdate().serialize(this, type.getNestedType()); + } else if (superType instanceof WeightedSetDataType) { + writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); + putInt(null, update.getUpdate().getValueUpdateClassID().id); + update.getUpdate().serialize(this, DataType.INT); + } else { + throw new SerializationException("MapValueUpdate only works for arrays and weighted sets"); + } + } + + @Override + public void write(ArithmeticValueUpdate update) { + putInt(null, update.getOperator().id); + putDouble(null, update.getOperand().doubleValue()); + } + + @Override + public void write(AssignValueUpdate update, DataType superType) { + if (update.getValue() == null) { + putByte(null, (byte) 0); + } else { + putByte(null, (byte) 1); + writeValue(this, superType, update.getValue()); + } + } + + @Override + public void write(RemoveValueUpdate update, DataType superType) { + writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); + } + + @Override + public void write(ClearValueUpdate clearValueUpdate, DataType superType) { + //TODO: This has never ever been implemented. Has this ever worked? + } + + /** + * Returns the serialized size of the given {@link Document}. Please note that this method performs actual + * serialization of the document, but simply return the size of the final {@link GrowableByteBuffer}. If you need + * the buffer itself, do NOT use this method. + * + * @param doc The Document whose size to calculate. + * @return The size in bytes. + */ + public static long getSerializedSize(Document doc) { + DocumentSerializer serializer = new VespaDocumentSerializerHead(new GrowableByteBuffer()); + serializer.write(doc); + return serializer.getBuf().position(); + } + + private static void writeValue(VespaDocumentSerializer42 serializer, DataType dataType, Object value) { + FieldValue fieldValue; + if (value instanceof FieldValue) { + fieldValue = (FieldValue)value; + } else { + fieldValue = dataType.createFieldValue(value); + } + fieldValue.serialize(serializer); + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java new file mode 100644 index 00000000000..7f4c1960122 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.io.GrowableByteBuffer; + +/** + * Class used for serializing documents on the current head document format. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class VespaDocumentSerializerHead extends VespaDocumentSerializer42 { + + public VespaDocumentSerializerHead(GrowableByteBuffer buf) { + super(buf); + } + + @Override + public void write(DocumentUpdate update) { + update.getId().serialize(this); + + update.getDocumentType().serialize(this); + + putInt(null, update.getFieldUpdates().size()); + + for (FieldUpdate up : update.getFieldUpdates()) { + up.serialize(this); + } + + DocumentUpdateFlags flags = new DocumentUpdateFlags(); + flags.setCreateIfNonExistent(update.getCreateIfNonExistent()); + putInt(null, flags.injectInto(update.getFieldPathUpdates().size())); + + for (FieldPathUpdate up : update.getFieldPathUpdates()) { + up.serialize(this); + } + } + + public void write(FieldPathUpdate update) { + putByte(null, (byte)update.getUpdateType().getCode()); + put(null, update.getOriginalFieldPath()); + put(null, update.getOriginalWhereClause()); + } + + public void write(AssignFieldPathUpdate update) { + write((FieldPathUpdate)update); + byte flags = 0; + if (update.getRemoveIfZero()) { + flags |= AssignFieldPathUpdate.REMOVE_IF_ZERO; + } + if (update.getCreateMissingPath()) { + flags |= AssignFieldPathUpdate.CREATE_MISSING_PATH; + } + if (update.isArithmetic()) { + flags |= AssignFieldPathUpdate.ARITHMETIC_EXPRESSION; + putByte(null, flags); + put(null, update.getExpression()); + } else { + putByte(null, flags); + update.getFieldValue().serialize(this); + } + } + + public void write(AddFieldPathUpdate update) { + write((FieldPathUpdate)update); + update.getNewValues().serialize(this); + } +} diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java new file mode 100644 index 00000000000..ffe6073cd6c --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -0,0 +1,314 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.Map; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.objects.FieldBase; +import com.yahoo.vespa.objects.Serializer; + +// TODO: Just inline all use of XmlSerializationHelper when the toXml methods in FieldValue subclasses are to be removed +// TODO: More cleanup, the put() methods generate a lot of superfluous objects (write should call put, not the other way around) +// TODO: remove pingpong between XmlSerializationHelper and FieldValue, this will go away when the toXml methods go away +/** + * Render a Document instance as XML. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@SuppressWarnings("deprecation") +public final class XmlDocumentWriter implements DocumentWriter { + + private final String indent; + private XmlStream buffer; + private Deque<FieldBase> optionalWrapperMarker = new ArrayDeque<FieldBase>(); + + public static XmlDocumentWriter createWriter(String indent) { + return new XmlDocumentWriter(indent); + } + + public XmlDocumentWriter() { + this(" "); + } + + private XmlDocumentWriter(String indent) { + this.indent = indent; + } + + // this method is silly, what is the intended way of doing this? + @Override + public void write(FieldBase field, FieldValue value) { + Class<?> valueType = value.getClass(); + if (valueType == AnnotationReference.class) { + write(field, (AnnotationReference) value); + } else if (valueType == Array.class) { + write(field, (Array<?>) value); + } else if (valueType == WeightedSet.class) { + write(field, (WeightedSet<?>) value); + } else if (valueType == Document.class) { + write(field, (Document) value); + } else if (valueType == Struct.class) { + write(field, (Struct) value); + } else if (valueType == ByteFieldValue.class) { + write(field, (ByteFieldValue) value); + } else if (valueType == DoubleFieldValue.class) { + write(field, (DoubleFieldValue) value); + } else if (valueType == FloatFieldValue.class) { + write(field, (FloatFieldValue) value); + } else if (valueType == IntegerFieldValue.class) { + write(field, (IntegerFieldValue) value); + } else if (valueType == LongFieldValue.class) { + write(field, (LongFieldValue) value); + } else if (valueType == Raw.class) { + write(field, (Raw) value); + } else if (valueType == PredicateFieldValue.class) { + write(field, (PredicateFieldValue) value); + } else if (valueType == StringFieldValue.class) { + write(field, (StringFieldValue) value); + } else { + throw new UnsupportedOperationException("Cannot serialize a " + + valueType.getName()); + } + } + + @Override + public void write(FieldBase field, Document value) { + buffer.beginTag("document"); + buffer.addAttribute("documenttype", value.getDataType().getName()); + buffer.addAttribute("documentid", value.getId()); + final java.lang.Long lastModified = value.getLastModified(); + if (lastModified != null) { + buffer.addAttribute("lastmodifiedtime", lastModified); + } + write(null, value.getHeader()); + write(null, value.getBody()); + + buffer.endTag(); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, Array<T> value) { + buffer.beginTag(field.getName()); + XmlSerializationHelper.printArrayXml(value, buffer); + buffer.endTag(); + } + + private void singleValueTag(FieldBase field, FieldValue value) { + buffer.beginTag(field.getName()); + value.printXml(buffer); + buffer.endTag(); + } + + @Override + public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) { + // TODO Auto-generated method stub + buffer.beginTag(field.getName()); + XmlSerializationHelper.printMapXml(map, buffer); + buffer.endTag(); + } + + @Override + public void write(FieldBase field, ByteFieldValue value) { + singleValueTag(field, value); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, + CollectionFieldValue<T> value) { + buffer.beginTag(field.getName()); + for (@SuppressWarnings("unchecked") + Iterator<FieldValue> i = (Iterator<FieldValue>) value.iterator(); + i.hasNext();) { + buffer.beginTag("item"); + i.next().printXml(buffer); + buffer.endTag(); + } + buffer.endTag(); + } + + @Override + public void write(FieldBase field, DoubleFieldValue value) { + singleValueTag(field, value); + } + + @Override + public void write(FieldBase field, FloatFieldValue value) { + singleValueTag(field, value); + } + + @Override + public void write(FieldBase field, IntegerFieldValue value) { + singleValueTag(field, value); + } + + @Override + public void write(FieldBase field, LongFieldValue value) { + singleValueTag(field, value); + } + + @Override + public void write(FieldBase field, Raw value) { + buffer.beginTag(field.getName()); + XmlSerializationHelper.printRawXml(value, buffer); + buffer.endTag(); + } + + @Override + public void write(FieldBase field, PredicateFieldValue value) { + singleValueTag(field, value); + } + + @Override + public void write(FieldBase field, StringFieldValue value) { + buffer.beginTag(field.getName()); + XmlSerializationHelper.printStringXml(value, buffer); + buffer.endTag(); + } + + @Override + public void write(FieldBase field, TensorFieldValue value) { + throw new IllegalArgumentException("write() for tensor field value not implemented yet"); + } + + private void optionalWrapperStart(FieldBase field) { + if (field == null) { + return; + } + + optionalWrapperMarker.addFirst(field); + + buffer.beginTag(field.getName()); + } + + private void optionalWrapperEnd(FieldBase field) { + if (field == null) { + return; + } + + if (optionalWrapperMarker.removeFirst() != field) { + throw new IllegalStateException("Unbalanced optional wrapper tags."); + } + + buffer.endTag(); + } + + @Override + public void write(FieldBase field, Struct value) { + optionalWrapperStart(field); + XmlSerializationHelper.printStructXml(value, buffer); + optionalWrapperEnd(field); + } + + @Override + public void write(FieldBase field, StructuredFieldValue value) { + buffer.beginTag(field.getName()); + Iterator<Map.Entry<Field, FieldValue>> i = value.iterator(); + while (i.hasNext()) { + Map.Entry<Field, FieldValue> v = i.next(); + buffer.beginTag(v.getKey().getName()); + v.getValue().printXml(buffer); + buffer.endTag(); + } + buffer.endTag(); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, + WeightedSet<T> value) { + buffer.beginTag(field.getName()); + XmlSerializationHelper.printWeightedSetXml(value, buffer); + buffer.endTag(); + } + + @Override + public void write(FieldBase field, AnnotationReference value) { + // TODO Auto-generated method stub + + } + + @Override + public Serializer putByte(FieldBase field, byte value) { + singleValueTag(field, new ByteFieldValue(value)); + return this; + } + + @Override + public Serializer putShort(FieldBase field, short value) { + singleValueTag(field, new IntegerFieldValue(value)); + return this; + } + + @Override + public Serializer putInt(FieldBase field, int value) { + singleValueTag(field, new IntegerFieldValue(value)); + return this; + } + + @Override + public Serializer putLong(FieldBase field, long value) { + singleValueTag(field, new LongFieldValue(value)); + return this; + } + + @Override + public Serializer putFloat(FieldBase field, float value) { + singleValueTag(field, new FloatFieldValue(value)); + return this; + } + + @Override + public Serializer putDouble(FieldBase field, double value) { + singleValueTag(field, new DoubleFieldValue(value)); + return this; + } + + @Override + public Serializer put(FieldBase field, byte[] value) { + write(field, new Raw(value)); + return this; + } + + @Override + public Serializer put(FieldBase field, ByteBuffer value) { + write(field, new Raw(value)); + return this; + } + + @Override + public Serializer put(FieldBase field, String value) { + write(field, new StringFieldValue(value)); + return this; + } + + @Override + public void write(Document document) { + buffer = new XmlStream(); + buffer.setIndent(indent); + optionalWrapperMarker.clear(); + write(new Field(document.getDataType().getName(), 0, document.getDataType(), true), document); + } + + @Override + public void write(DocumentId id) { + throw new UnsupportedOperationException("Writing a DocumentId as XML is not implemented."); + } + + @Override + public void write(DocumentType type) { + throw new UnsupportedOperationException("Writing a DocumentId as XML is not implemented."); + + } + + public String lastRendered() { + return buffer.toString(); + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java new file mode 100644 index 00000000000..d7362095cf3 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.document.Document; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.*; +import com.yahoo.text.Utf8; +import org.apache.commons.codec.binary.Base64; + +import java.io.UnsupportedEncodingException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Date: Apr 17, 2008 + * + * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> + */ +public class XmlSerializationHelper { + + public static void printArrayXml(Array array, XmlStream xml) { + List<FieldValue> lst = array.getValues(); + for (FieldValue value : lst) { + xml.beginTag("item"); + value.printXml(xml); + xml.endTag(); + } + } + + public static <K extends FieldValue, V extends FieldValue> void printMapXml(MapFieldValue<K, V> map, XmlStream xml) { + for (Map.Entry<K, V> e : map.entrySet()) { + FieldValue key = e.getKey(); + FieldValue val = e.getValue(); + xml.beginTag("item"); + xml.beginTag("key"); + key.printXml(xml); + xml.endTag(); + xml.beginTag("value"); + val.printXml(xml); + xml.endTag(); + xml.endTag(); + } + } + + public static void printByteXml(ByteFieldValue b, XmlStream xml) { + xml.addContent(b.toString()); + } + + public static void printDocumentXml(Document doc, XmlStream xml) { + xml.addAttribute("documenttype", doc.getDataType().getName()); + xml.addAttribute("documentid", doc.getId()); + final java.lang.Long lastModified = doc.getLastModified(); + if (lastModified != null) { + xml.addAttribute("lastmodifiedtime", lastModified); + } + doc.getHeader().printXml(xml); + doc.getBody().printXml(xml); + } + + public static void printDoubleXml(DoubleFieldValue d, XmlStream xml) { + xml.addContent(d.toString()); + } + + public static void printFloatXml(FloatFieldValue f, XmlStream xml) { + xml.addContent(f.toString()); + } + + public static void printIntegerXml(IntegerFieldValue f, XmlStream xml) { + xml.addContent(f.toString()); + } + + public static void printLongXml(LongFieldValue l, XmlStream xml) { + xml.addContent(l.toString()); + } + + public static void printRawXml(Raw r, XmlStream xml) { + xml.addAttribute("binaryencoding", "base64"); + xml.addContent(new Base64(0).encodeToString(r.getByteBuffer().array())); + } + + public static void printStringXml(StringFieldValue s, XmlStream xml) { + String content = s.getString(); + if (containsNonPrintableCharactersString(content)) { + byte[] bytecontent = Utf8.toBytes(content); + xml.addAttribute("binaryencoding", "base64"); + xml.addContent(new Base64(0).encodeToString(bytecontent)); + } else { + xml.addContent(content); + } + } + + public static void printStructXml(Struct s, XmlStream xml) { + Iterator<Map.Entry<Field, FieldValue>> it = s.iterator(); + while (it.hasNext()) { + Map.Entry<Field, FieldValue> val = it.next(); + xml.beginTag(val.getKey().getName()); + val.getValue().printXml(xml); + xml.endTag(); + } + } + + public static void printWeightedSetXml(WeightedSet ws, XmlStream xml) { + Iterator<FieldValue> it = ws.fieldValueIterator(); + while (it.hasNext()) { + FieldValue val = it.next(); + xml.beginTag("item"); + xml.addAttribute("weight", ws.get(val)); + val.printXml(xml); + xml.endTag(); + } + } + + private static boolean containsNonPrintableCharactersByte(final byte[] buffer) { + for (byte b : buffer) { + if (b < 32 && (b != 9 && b != 10 && b != 13)) return true; + } + return false; + } + + private static boolean containsNonPrintableCharactersString(final CharSequence buffer) { + for (int i = 0; i < buffer.length(); i++) { + char b = buffer.charAt(i); + if (b < 32 && (b != 9 && b != 10 && b != 13)) return true; + } + return false; + } +} diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlStream.java b/document/src/main/java/com/yahoo/document/serialization/XmlStream.java new file mode 100644 index 00000000000..f0c8451cd2a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/XmlStream.java @@ -0,0 +1,215 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.text.XML; + +import java.io.StringWriter; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.ListIterator; + +/** + * Class for writing XML in a simplified way. + * <p> + * Give a writer for it to write the XML directly to. If none is given a + * StringWriter is used so you can call toString() on the class to get the + * XML. + * <p> + * You build XML by calling beginTag(name), addAttribute(id, value), endTag(). + * Remember to close all your tags, or you'll get an exception when calling + * toString(). If writing directly to a writer, call isFinalized to verify that + * all tags have been closed. + * <p> + * The XML escaping tools only give an interface for escape from and to a string + * value. Thus writing of all data here is also just available through strings. + * + * @author <a href="humbe@yahoo-inc.com">Haakon Humberset</a> + */ +public class XmlStream { + + // Utility class to hold attributes internally until it's time to write them + private static class Attribute { + final String name; + final String value; + + public Attribute(String name, Object value) { + this.name = name; + this.value = value.toString(); + } + } + + private final StringWriter writer; // Writer to output XML to. + private final Deque<String> tags = new ArrayDeque<String>(); + private String indent = ""; + // We write tags lazily for several reasons: + // - To allow recursive methods to have both parents and child add + // attributes to last tag, without giving child responsibility of + // closing or creating the tag. + // - Be able to check content before adding whitespace, such that we + // can add newlines if content is new tags for instance. + // The cached variables here will be written with the flush() method. + private String cachedTag = null; + private final List<Attribute> cachedAttribute = new ArrayList<Attribute>(); + private final List<String> cachedContent = new ArrayList<String>(); + + /** + * Create an XmlStream writing to a StringWriter. + * Fetch XML through toString() once you're done creating it. + */ + public XmlStream() { + writer = new StringWriter(); + } + + /** + * Set an indent to use for pretty printing of XML. Default is no indent. + * + * @param indent the initial indentation + */ + public void setIndent(String indent) { + this.indent = indent; + } + + /** + * Check if all tags have been properly closed. + * + * @return true if all tags are closed + */ + public boolean isFinalized() { + return (tags.isEmpty() && cachedTag == null); + } + + public String toString() { + if (!isFinalized()) { + throw new IllegalStateException("There are still" + " tag(s) that are not closed."); + } + StringWriter sw = writer; // Ensure we have string writer + return sw.toString(); + } + + /** + * Add a new XML tag with the given name. + * + * @param name the tag name + */ + public void beginTag(String name) { + if (!XML.isName(name)) { + throw new IllegalArgumentException("The name '" + name + + "' cannot be used as an XML tag name. Legal names must adhere to" + + "http://www.w3.org/TR/2006/REC-xml11-20060816/#sec-common-syn"); + } + if (cachedTag != null) flush(false); + cachedTag = name; + } + + /** + * Add a new XML attribute to the last tag started. + * The tag cannot already have had content added to it, or been ended. + * If a null value is added, the attribute will be skipped. + * + * @param key the attribute name + * @param value the attribute value + */ + public void addAttribute(String key, Object value) { + if (value == null) { + return; + } + if (cachedTag == null) { + throw new IllegalStateException("There is no open tag to add attributes to."); + } + if (!XML.isName(key)) { + throw new IllegalArgumentException("The name '" + key + + "' cannot be used as an XML attribute name. Legal names must adhere to" + + " http://www.w3.org/TR/2006/REC-xml11-20060816/#sec-common-syn"); + } + cachedAttribute.add(new Attribute(key, value)); + } + + /** + * Add content to the last tag. + * + * @param content the content to add to the last tag + */ + public void addContent(String content) { + if (cachedTag != null) { + cachedContent.add(XML.xmlEscape(content, false)); + } else if (tags.isEmpty()) { + throw new IllegalStateException("There is no open tag to add content to."); + } else { + for (int i = 0; i < tags.size(); ++i) { + writer.write(indent); + } + writer.write(XML.xmlEscape(content, false)); + writer.write('\n'); + } + } + + /** + * Ends the last tag created. + * + */ + public void endTag() { + if (cachedTag != null) { + flush(true); + } else if (tags.isEmpty()) { + throw new IllegalStateException("Cannot end non-existing tag"); + } else { + for (int i = 1; i < tags.size(); ++i) { + writer.write(indent); + } + writer.write("</"); + writer.write(tags.removeFirst()); + writer.write(">\n"); + } + } + + // Utility function to write whatever is cached. + private void flush(boolean endTag) { + if (cachedTag == null) { + throw new IllegalStateException("Cannot write non-existing tag"); + } + for (int i = 0; i < tags.size(); ++i) { + writer.write(indent); + } + writer.write('<'); + writer.write(cachedTag); + for (ListIterator<Attribute> it = cachedAttribute.listIterator(); it.hasNext();) { + Attribute attr = it.next(); + writer.write(' '); + writer.write(attr.name); + writer.write("=\""); + writer.write(XML.xmlEscape(attr.value, true)); + writer.write('"'); + } + cachedAttribute.clear(); + if (cachedContent.isEmpty() && endTag) { + writer.write("/>\n"); + } else if (cachedContent.isEmpty()) { + writer.write(">\n"); + tags.addFirst(cachedTag); + } else { + writer.write(">"); + if (!endTag) { + writer.write('\n'); + for (int i = 0; i <= tags.size(); ++i) { + writer.write(indent); + } + } + for (String content : cachedContent) { + writer.write(content); + } + cachedContent.clear(); + if (endTag) { + writer.write("</"); + writer.write(cachedTag); + writer.write(">\n"); + } else { + writer.write('\n'); + tags.addFirst(cachedTag); + } + } + cachedTag = null; + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/package-info.java b/document/src/main/java/com/yahoo/document/serialization/package-info.java new file mode 100644 index 00000000000..711990c5d9a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.serialization; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/update/AddValueUpdate.java b/document/src/main/java/com/yahoo/document/update/AddValueUpdate.java new file mode 100644 index 00000000000..1ad2941c80c --- /dev/null +++ b/document/src/main/java/com/yahoo/document/update/AddValueUpdate.java @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.update; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.serialization.DocumentUpdateWriter; + +/** + * <p>Value update representing an addition of a value (possibly with an associated weight) + * to a multi-valued data type.</p> + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class AddValueUpdate extends ValueUpdate { + protected FieldValue value; + protected Integer weight; + + AddValueUpdate(FieldValue value) { + super(ValueUpdateClassID.ADD); + setValue(value, 1); + } + + public AddValueUpdate(FieldValue key, int weight) { + super(ValueUpdateClassID.ADD); + setValue(key, weight); + } + + private void setValue(FieldValue key, int weight) { + this.value = key; + this.weight = weight; + } + + /** + * Returns the value of this value update. + * + * @return the value of this ValueUpdate + * @see com.yahoo.document.DataType + */ + public FieldValue getValue() { return value; } + + public void setValue(FieldValue value) { this.value=value; } + + /** + * Return the associated weight of this value update. + * + * @return the weight of this value update, or 1 if unset + */ + public int getWeight() { + return weight; + } + + @Override + public FieldValue applyTo(FieldValue val) { + if (val instanceof WeightedSet) { + WeightedSet wset = (WeightedSet) val; + wset.put((FieldValue) value, weight); + } else if (val instanceof CollectionFieldValue) { + CollectionFieldValue fval = (CollectionFieldValue) val; + fval.add((FieldValue) value); + } else { + throw new IllegalStateException("Cannot add "+value+" to field of type " + val.getClass().getName()); + } + return val; + } + + @Override + protected void checkCompatibility(DataType fieldType) { + if (!(fieldType instanceof CollectionDataType)) { + throw new UnsupportedOperationException("Expected collection, got " + fieldType.getName() + "."); + } + fieldType = ((CollectionDataType)fieldType).getNestedType(); + if (value != null && !value.getDataType().equals(fieldType)) { + throw new IllegalArgumentException("Expected " + fieldType.getName() + ", got " + + value.getDataType().getName()); + } + } + + @Override + public void serialize(DocumentUpdateWriter data, DataType superType) { + data.write(this, superType); + } + + @Override + public boolean equals(Object o) { + return o instanceof AddValueUpdate && super.equals(o) && value.equals(((AddValueUpdate) o).value) && + weight.equals(((AddValueUpdate) o).weight); + } + + @Override + public int hashCode() { + return super.hashCode() + value.hashCode() + weight; + } + + @Override + public String toString() { + return super.toString() + " " + value + " " + weight; + } + +} diff --git a/document/src/main/java/com/yahoo/document/update/ArithmeticValueUpdate.java b/document/src/main/java/com/yahoo/document/update/ArithmeticValueUpdate.java new file mode 100644 index 00000000000..c4e677cab66 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/update/ArithmeticValueUpdate.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.update; + +import com.yahoo.document.DataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.NumericFieldValue; +import com.yahoo.document.serialization.DocumentUpdateWriter; + +/** + * <p>Value update representing an arithmetic operation on a numeric data type.</p> + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ArithmeticValueUpdate extends ValueUpdate<DoubleFieldValue> { + protected Operator operator; + protected DoubleFieldValue operand; + + public ArithmeticValueUpdate(Operator operator, DoubleFieldValue operand) { + super(ValueUpdateClassID.ARITHMETIC); + this.operator = operator; + this.operand = operand; + } + + public ArithmeticValueUpdate(Operator operator, Number operand) { + this(operator, new DoubleFieldValue(operand.doubleValue())); + } + + /** + * Returns the operator of this arithmatic value update. + * + * @return the operator + * @see com.yahoo.document.update.ArithmeticValueUpdate.Operator + */ + public Operator getOperator() { + return operator; + } + + /** + * Returns the operand of this arithmetic value update. + * + * @return the operand + */ + public Number getOperand() { + return operand.getDouble(); + } + + /** Returns the operand */ + public DoubleFieldValue getValue() { return operand; } + + /** Sets the operand */ + public void setValue(DoubleFieldValue value) { operand=value; } + + @Override + public FieldValue applyTo(FieldValue oldValue) { + if (oldValue instanceof NumericFieldValue) { + Number number = (Number) oldValue.getWrappedValue(); + oldValue.assign(calculate(number)); + } else { + throw new IllegalStateException("Cannot use arithmetic value update on non-numeric datatype "+oldValue.getClass().getName()); + } + return oldValue; + } + + @Override + protected void checkCompatibility(DataType fieldType) { + if (!(fieldType instanceof NumericDataType)) { + throw new UnsupportedOperationException("Expected numeric type, got " + fieldType.getName() + "."); + } + } + + private double calculate(Number operand2) { + switch (operator) { + case ADD: + return operand2.doubleValue() + operand.getDouble(); + case DIV: + return operand2.doubleValue() / operand.getDouble(); + case MUL: + return operand2.doubleValue() * operand.getDouble(); + case SUB: + return operand2.doubleValue() - operand.getDouble(); + } + return 0d; + } + + @Override + public void serialize(DocumentUpdateWriter data, DataType superType) { + data.write(this); + } + + @Override + public boolean equals(Object o) { + return o instanceof ArithmeticValueUpdate && super.equals(o) && + operator == ((ArithmeticValueUpdate) o).operator && operand.equals(((ArithmeticValueUpdate) o).operand); + } + + @Override + public int hashCode() { + return super.hashCode() + operator.hashCode() + operand.hashCode(); + } + + @Override + public String toString() { + return super.toString() + " " + operator.name + " " + operand; + } + + /** + * Lists valid operations that can be performed by an ArithmeticValueUpdate. + */ + public enum Operator { + /** + * Add the operand to the value. + */ + ADD(0, "add"), + /** + * Divide the value by the operand. + */ + DIV(1, "divide"), + /** + * Multiply the value by the operand. + */ + MUL(2, "multiply"), + /** + * Subtract the operand from the value. + */ + SUB(3, "subtract"); + + /** + * The numeric ID of the operator, used for serialization. + */ + public final int id; + /** + * The name of the operator, mainly used in toString() methods. + */ + public final String name; + + Operator(int id, String name) { + this.id = id; + this.name = name; + } + + /** + * Returns the operator with the specified ID. + * + * @param id the ID to search for + * @return the Operator with the specified ID, or null if it does not exist. + */ + public static Operator getID(int id) { + for (Operator operator : Operator.values()) { + if (operator.id == id) { + return operator; + } + } + return null; + } + } +} + diff --git a/document/src/main/java/com/yahoo/document/update/AssignValueUpdate.java b/document/src/main/java/com/yahoo/document/update/AssignValueUpdate.java new file mode 100644 index 00000000000..f1157d272e2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/update/AssignValueUpdate.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.update; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.serialization.DocumentUpdateWriter; + +/** + * <p>Value update that represents assigning a new value.</p> + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class AssignValueUpdate extends ValueUpdate { + protected FieldValue value; + + public AssignValueUpdate(FieldValue value) { + super(ValueUpdateClassID.ASSIGN); + this.value = value; + } + + /** + * <p>Returns the value of this value update.</p> + * + * <p>The type of the value is defined by the type of this field + * in this documents DocumentType - a java.lang primitive wrapper for single value types, + * java.util.List for arrays and {@link com.yahoo.document.datatypes.WeightedSet WeightedSet} + * for weighted sets.</p> + * + * @return the value of this ValueUpdate + * @see com.yahoo.document.DataType + */ + public FieldValue getValue() { return value; } + + /** + * <p>Sets the value to assign.</p> + * + * <p>The type of the value must match the type of this field + * in this documents DocumentType - a java.lang primitive wrapper for single value types, + * java.util.List for arrays and {@link com.yahoo.document.datatypes.WeightedSet WeightedSet} + * for weighted sets.</p> + */ + public void setValue(FieldValue value) { this.value=value; } + + @Override + public FieldValue applyTo(FieldValue fval) { + if (value == null) return null; + fval.assign(value); + return fval; + } + + @Override + protected void checkCompatibility(DataType fieldType) { + if (value != null && !value.getDataType().equals(fieldType)) { + throw new IllegalArgumentException("Expected " + fieldType.getName() + ", got " + + value.getDataType().getName()); + } + } + + @Override + public void serialize(DocumentUpdateWriter data, DataType superType) { + data.write(this, superType); + } + + @Override + public boolean equals(Object o) { + boolean baseEquals = o instanceof AssignValueUpdate && super.equals(o); + + if (!baseEquals) return false; + + if (value == null && ((AssignValueUpdate) o).value == null) { + return true; + } else if (value != null && value.equals(((AssignValueUpdate) o).value)) { + return true; + } + return false; + } + + @Override + public int hashCode() { + return super.hashCode() + (value == null ? 0 : value.hashCode()); + } + + @Override + public String toString() { + return super.toString() + " " + value; + } +} diff --git a/document/src/main/java/com/yahoo/document/update/ClearValueUpdate.java b/document/src/main/java/com/yahoo/document/update/ClearValueUpdate.java new file mode 100644 index 00000000000..a9a0ad271ef --- /dev/null +++ b/document/src/main/java/com/yahoo/document/update/ClearValueUpdate.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.update; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.serialization.DocumentUpdateWriter; + +/** + * <p>Value update that represents clearing a field. Clearing a field mean removing it.</p> + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ClearValueUpdate extends ValueUpdate { + + public ClearValueUpdate() { + super(ValueUpdateClassID.CLEAR); + } + + @Override + public FieldValue applyTo(FieldValue fval) { + return null; + } + + @Override + protected void checkCompatibility(DataType fieldType) { + // empty + } + + @Override + public FieldValue getValue() { + return null; + } + + @Override + public void setValue(FieldValue value) { + // empty + } + + @Override + public void serialize(DocumentUpdateWriter data, DataType superType) { + data.write(this, superType); + } +} diff --git a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java new file mode 100644 index 00000000000..c4250a33f28 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java @@ -0,0 +1,624 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.update; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.document.serialization.DocumentUpdateReader; +import com.yahoo.document.serialization.DocumentUpdateWriter; +import com.yahoo.io.GrowableByteBuffer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * <p>A field update holds a list of value updates that will be applied atomically to a field in a document.</p> + * <p>To create a field update that contains only a single value update, use the static factory methods provided by this + * class.</p> <p>Example:</p> + * <pre> + * FieldUpdate clearFieldUpdate = FieldUpdate.createClearField(field); + * </pre> + * <p>It is also possible to create a field update that holds more than one value update.</p> + * <p>Example:</p> + * <pre> + * FieldUpdate fieldUpdate = FieldUpdate.create(field); + * ValueUpdate incrementValue = ValueUpdate.createIncrement("foo", 130d); + * ValueUpdate addValue = ValueUpdate.createAdd("bar", 100); + * fieldUpdate.addValueUpdate(incrementValue); + * fieldUpdate.addValueUpdate(addValue);</pre> + * <p>Note that the addValueUpdate() method returns a reference to itself to support chaining, so the last two + * lines could be written as one:</p> + * <pre> + * fieldUpdate.addValueUpdate(incrementValue).addValueUpdate(addValue); + * </pre> + * <p>Note also that the second example above is equivalent to:</p> + * <pre> + * FieldUpdate fieldUpdate = FieldUpdate.createIncrement(field, "foo", 130d); + * ValueUpdate addValue = ValueUpdate.createAdd("bar", 100); + * fieldUpdate.addValueUpdate(addValue); + * </pre> + * <p>Note that even though updates take fields as arguments, those fields are not necessarily a field of a document + * type - any name/value pair which existing in an updatable structure can be addressed by creating the Fields as + * needed. For example: + * <pre> + * FieldUpdate field=FieldUpdate.createIncrement(new Field("myattribute",DataType.INT),130); + * </pre> + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @see com.yahoo.document.update.ValueUpdate + * @see com.yahoo.document.DocumentUpdate + */ +public class FieldUpdate { + + protected Field field; + protected List<ValueUpdate> valueUpdates = new ArrayList<>(); + + // Used only while deserializing. + private DocumentType documentType = null; + private int serializationVersion = 0; + + FieldUpdate(Field field) { + this.field = field; + } + + FieldUpdate(Field field, ValueUpdate valueUpd) { + this(field); + addValueUpdate(valueUpd); + } + + FieldUpdate(Field field, List<ValueUpdate> valueUpdates) { + this(field); + addValueUpdates(valueUpdates); + } + + public FieldUpdate(DocumentUpdateReader reader, DocumentType type, int serializationVersion) { + documentType = type; + this.serializationVersion = serializationVersion; + reader.read(this); + } + + public DocumentType getDocumentType() { + return documentType; + } + + public int getSerializationVersion() { + return serializationVersion; + } + + /** Returns the field that this field update applies to */ + public Field getField() { + return field; + } + + /** + * Sets the field this update applies to. Note that this does not need to be + * a field of the document type in question - a field is just the name and type + * of some value to be updated. + */ + public void setField(Field field) { + this.field = field; + } + + /** + * Applies this field update. + * + * @param doc the document to apply the update to + * @return a reference to itself + */ + public FieldUpdate applyTo(Document doc) { + for (ValueUpdate vupd : valueUpdates) { + DataType dataType = field.getDataType(); + FieldValue oldValue = doc.getFieldValue(field); + boolean existed = (oldValue != null); + + if (!existed) { + oldValue = dataType.createFieldValue(); + } + + FieldValue newValue = vupd.applyTo(oldValue); + + if (newValue == null) { + if (existed) { + doc.removeFieldValue(field); + } + } else { + doc.setFieldValue(field, newValue); + } + } + return this; + } + + /** + * Adds a value update to the list of value updates. + * + * @param valueUpdate the ValueUpdate to add + * @return a reference to itself + * @throws IllegalArgumentException if the data type of the value update is not equal to the data type of this field + */ + public FieldUpdate addValueUpdate(ValueUpdate valueUpdate) { + valueUpdate.checkCompatibility(field.getDataType()); //will throw exception + valueUpdates.add(valueUpdate); + return this; + } + + /** + * Adds a value update to the list of value updates. + * + * @param index the index where this value update should be added + * @param valueUpdate the ValueUpdate to add + * @return a reference to itself + * @throws IllegalArgumentException if the data type of the value update is not equal to the data type of this field + */ + public FieldUpdate addValueUpdate(int index, ValueUpdate valueUpdate) { + valueUpdate.checkCompatibility(field.getDataType()); //will throw exception + valueUpdates.add(index, valueUpdate); + return this; + } + + /** + * Adds a list of value updates to the list of value updates. + * + * @param valueUpdates a list containing the value updates to add + * @return a reference to itself + * @throws IllegalArgumentException if the data type of the value update is not equal to the data type of this field + */ + public FieldUpdate addValueUpdates(List<ValueUpdate> valueUpdates) { + for (ValueUpdate vupd : valueUpdates) { + addValueUpdate(vupd); + } + return this; + } + + /** + * Removes the value update at the specified position in the list of value updates. + * + * @param index the index of the ValueUpdate to remove + * @return the ValueUpdate previously at the specified position + * @throws IndexOutOfBoundsException if index is out of range + */ + public ValueUpdate removeValueUpdate(int index) { + return valueUpdates.remove(index); + } + + /** + * Replaces the value update at the specified position in the list of value updates + * with the specified value update. + * + * @param index index of value update to replace + * @param update value update to be stored at the specified position + * @return the value update previously at the specified position + * @throws IndexOutOfBoundsException if index out of range (index < 0 || index >= size()) + */ + public ValueUpdate setValueUpdate(int index, ValueUpdate update) { + return valueUpdates.set(index, update); + } + + /** + * Get the number of value updates in this field update. + * + * @return the size of the List of FieldUpdates + */ + public int size() { + return valueUpdates.size(); + } + + /** Removes all value updates from the list of value updates. */ + public void clearValueUpdates() { + valueUpdates.clear(); + } + + /** + * Get the value update at the specified index in the list of value updates. + * + * @param index the index of the ValueUpdate to return + * @return the ValueUpdate at the specified index + * @throws IndexOutOfBoundsException if index is out of range + */ + public ValueUpdate getValueUpdate(int index) { + return valueUpdates.get(index); + } + + /** + * Get an unmodifiable list of all value updates that this field update specifies. + * + * @return a list of all ValueUpdates in this FieldUpdate + */ + public List<ValueUpdate> getValueUpdates() { + return Collections.unmodifiableList(valueUpdates); + } + + /** + * Get value updates with the specified valueUpdateClassID. The caller gets ownership of the returned list, and + * subsequent modifications to the list does not change the state of this object. + * + * @param classID the classID of ValueUpdates to return + * @return a List of ValueUpdates of the specified classID (possibly empty, but not null) + */ + public List<ValueUpdate> getValueUpdates(ValueUpdate.ValueUpdateClassID classID) { + List<ValueUpdate> updateList = new ArrayList<>(); + for (ValueUpdate vupd : valueUpdates) { + if (vupd.getValueUpdateClassID() == classID) { + updateList.add(vupd); + } + } + return updateList; + } + + /** + * Returns whether this field update contains (at least) one update of the given type + * + * @param classID the classID of ValueUpdates to check for + * @return true if there is at least one value update of the given type in this + */ + public boolean hasValueUpdate(ValueUpdate.ValueUpdateClassID classID) { + for (ValueUpdate vupd : valueUpdates) { + if (vupd.getValueUpdateClassID() == classID) { + return true; + } + } + return false; + } + + /** + * Returns whether or not this field update contains any value updates. + * + * @return True if this update is empty. + */ + public boolean isEmpty() { + return valueUpdates.isEmpty(); + } + + /** + * Adds all the {@link ValueUpdate}s of the given FieldUpdate to this. If the given FieldUpdate refers to a + * different {@link Field} than this, this method throws an exception. + * + * @param update The update whose content to add to this. + * @throws IllegalArgumentException If the {@link Field} of the given FieldUpdate does not match this. + */ + public void addAll(FieldUpdate update) { + if (update == null) { + return; + } + if (!field.equals(update.field)) { + throw new IllegalArgumentException("Expected " + field + ", got " + update.field + "."); + } + addValueUpdates(update.valueUpdates); + } + + public final void serialize(GrowableByteBuffer buf) { + serialize(DocumentSerializerFactory.create42(buf)); + } + + public void serialize(DocumentUpdateWriter data) { + data.write(this); + } + + @Override + public boolean equals(Object o) { + return o instanceof FieldUpdate && field.equals(((FieldUpdate)o).field) && + valueUpdates.equals(((FieldUpdate)o).valueUpdates); + } + + @Override + public int hashCode() { + return field.getId(Document.SERIALIZED_VERSION) + valueUpdates.hashCode(); + } + + @Override + public String toString() { + return "'" + field.getName() + "' " + valueUpdates; + } + + /** + * Creates a new, empty field update with no encapsulated value updates. Use this method to add an arbitrary + * set of value updates using the FieldUpdate.addValueUpdate() method. + * + * @param field the Field to alter + * @return a new, empty FieldUpdate + * @see com.yahoo.document.update.ValueUpdate + * @see FieldUpdate#addValueUpdate(ValueUpdate) + */ + public static FieldUpdate create(Field field) { + return new FieldUpdate(field); + } + + /** + * Creates a new field update that clears the field. This operation removes the value/field completely. + * + * @param field the Field to clear + * @return a FieldUpdate specifying the clearing + * @see com.yahoo.document.update.FieldUpdate#createClear(Field) + */ + public static FieldUpdate createClearField(Field field) { + return new FieldUpdate(field, ValueUpdate.createClear()); + } + + /** + * Creates a new field update, with one encapsulated value update + * specifying an addition of a value to an array or a key to a weighted set (with default weight 1). + * + * @param field the Field to add a value to + * @param value the value to add to the array, or key to add to the weighted set + * @return a FieldUpdate specifying the addition + * @throws IllegalArgumentException if the runtime type of newValue does not match the type required by field + * @throws UnsupportedOperationException if the field type is not array or weighted set + * @see com.yahoo.document.update.ValueUpdate#createAdd(FieldValue) + */ + public static FieldUpdate createAdd(Field field, FieldValue value) { + return new FieldUpdate(field, ValueUpdate.createAdd(value)); + } + + /** + * Creates a new field update, with one encapsulated value update + * specifying an addition of a key (with a specified weight) to a weighted set. If this + * method is used on an array data type, the weight will be omitted. + * + * @param field the Field to a add a key to + * @param key the key to add + * @param weight the weight to associate with the given key + * @return a FieldUpdate specifying the addition + * @throws IllegalArgumentException if the runtime type of key does not match the type required by field + * @throws UnsupportedOperationException if the field type is not array or weighted set + * @see com.yahoo.document.update.ValueUpdate#createAdd(FieldValue,Integer) + */ + public static FieldUpdate createAdd(Field field, FieldValue key, Integer weight) { + return new FieldUpdate(field, ValueUpdate.createAdd(key, weight)); + } + + /** + * Creates a new field update, with encapsulated value updates, + * specifying an addition of all values in a given list to an array. If this method is used on a weighted set data + * type, the default weights will be 1. + * + * @param field the Field to add an array of values to + * @param values a List containing the values to add + * @return a FieldUpdate specifying the addition + * @throws IllegalArgumentException if the runtime type of values does not match the type required by field + * @throws UnsupportedOperationException if the field type is not array or weighted set + * @see com.yahoo.document.update.ValueUpdate#createAddAll(java.util.List) + * @see ValueUpdate#createAdd(FieldValue) + */ + public static FieldUpdate createAddAll(Field field, List<? extends FieldValue> values) { + return new FieldUpdate(field, ValueUpdate.createAddAll(values)); + } + + /** + * Creates a new field update, with encapsulated value updates, + * specifying an addition of all key/weight pairs in a weighted set to a weighted set. If this method + * is used on an array data type, the weights will be omitted. + * + * @param field the Field to add the key/weight pairs in a weighted set to + * @param set a WeightedSet containing the key/weight pairs to add + * @return a FieldUpdate specifying the addition + * @throws IllegalArgumentException if the runtime type of values does not match the type required by field + * @throws UnsupportedOperationException if the field type is not weighted set or array + * @see ValueUpdate#createAdd(FieldValue, Integer) + * @see com.yahoo.document.update.ValueUpdate#createAddAll(com.yahoo.document.datatypes.WeightedSet) + */ + public static FieldUpdate createAddAll(Field field, WeightedSet<? extends FieldValue> set) { + return new FieldUpdate(field, ValueUpdate.createAddAll(set)); + } + + /** + * Creates a new field update, with one encapsulated value update that increments a value. + * Note that the data type must be a numeric type. + * + * @param field the field to increment the value of + * @param increment the number to increment by + * @return a FieldUpdate specifying the increment + * @throws UnsupportedOperationException if the data type is non-numeric + * @see ValueUpdate#createIncrement(Number) + */ + public static FieldUpdate createIncrement(Field field, Number increment) { + return new FieldUpdate(field, ValueUpdate.createIncrement(increment)); + } + + /** + * Creates a new field update, with one encapsulated value update that increments a weight in a weighted set. + * + * @param field the field to increment one of the weights of + * @param key the key whose weight in the weighted set to increment + * @param increment the number to increment by + * @return a FieldUpdate specifying the increment + * @throws IllegalArgumentException if key is not equal to the nested type of the weighted set + * @see ValueUpdate#createIncrement(Number) + * @see ValueUpdate#createMap(FieldValue, ValueUpdate) + */ + public static FieldUpdate createIncrement(Field field, FieldValue key, Number increment) { + return new FieldUpdate(field, ValueUpdate.createIncrement(key, increment)); + } + + /** + * Creates a new field update, with one encapsulated value update that decrements a value. + * Note that the data type must be a numeric type. + * + * @param field the field to decrement the value of + * @param decrement the number to decrement by + * @return a FieldUpdate specifying the decrement + * @throws UnsupportedOperationException if the data type is non-numeric + * @see ValueUpdate#createDecrement(Number) + */ + public static FieldUpdate createDecrement(Field field, Number decrement) { + return new FieldUpdate(field, ValueUpdate.createDecrement(decrement)); + } + + /** + * Creates a new field update, with one encapsulated value update that decrements a weight in a weighted set. + * + * @param field the field to decrement one of the weights of + * @param key the key whose weight in the weighted set to decrement + * @param decrement the number to decrement by + * @return a FieldUpdate specifying the decrement + * @throws IllegalArgumentException if key is not equal to the nested type of the weighted set + * @see ValueUpdate#createDecrement(Number) + * @see ValueUpdate#createMap(FieldValue, ValueUpdate) + */ + public static FieldUpdate createDecrement(Field field, FieldValue key, Number decrement) { + return new FieldUpdate(field, ValueUpdate.createDecrement(key, decrement)); + } + + /** + * Creates a new field update, with one encapsulated value update that multiplies a value. + * Note that the data type must be a numeric type. + * + * @param field the field to multiply the value of + * @param factor the number to multiply by + * @return a FieldUpdate specifying the multiplication + * @throws UnsupportedOperationException if the data type is non-numeric + * @see ValueUpdate#createMultiply(Number) + */ + public static FieldUpdate createMultiply(Field field, Number factor) { + return new FieldUpdate(field, ValueUpdate.createMultiply(factor)); + } + + /** + * Creates a new field update, with one encapsulated value update that multiplies a weight in a weighted set. + * + * @param field the field to multiply one of the weights of + * @param key the key whose weight in the weighted set to multiply + * @param factor the number to multiply by + * @return a FieldUpdate specifying the multiplication + * @throws IllegalArgumentException if key is not equal to the nested type of the weighted set + * @see ValueUpdate#createMultiply(Number) + * @see ValueUpdate#createMap(FieldValue, ValueUpdate) + */ + public static FieldUpdate createMultiply(Field field, FieldValue key, Number factor) { + return new FieldUpdate(field, ValueUpdate.createMultiply(key, factor)); + } + + /** + * Creates a new field update, with one encapsulated value update that divides a value. + * Note that the data type must be a numeric type. + * + * @param field the field to divide the value of + * @param divisor the number to divide by + * @return a FieldUpdate specifying the division + * @throws UnsupportedOperationException if the data type is non-numeric + * @see ValueUpdate#createDivide(Number) + */ + public static FieldUpdate createDivide(Field field, Number divisor) { + return new FieldUpdate(field, ValueUpdate.createDivide(divisor)); + } + + /** + * Creates a new field update, with one encapsulated value update that divides a weight in a weighted set. + * + * @param field the field to divide one of the weights of + * @param key the key whose weight in the weighted set to divide + * @param divisor the number to divide by + * @return a FieldUpdate specifying the division + * @throws IllegalArgumentException if key is not equal to the nested type of the weighted set + * @see ValueUpdate#createDivide(Number) + * @see ValueUpdate#createMap(FieldValue, ValueUpdate) + */ + public static FieldUpdate createDivide(Field field, FieldValue key, Number divisor) { + return new FieldUpdate(field, ValueUpdate.createDivide(key, divisor)); + } + + /** + * Creates a new field update, with one encapsulated value update, + * that assigns a new value, completely overwriting the previous value. Note that it is possible to pass + * newValue=null to this method to remove the value completely. + * + * @param field the Field to assign a new value to + * @param newValue the value to assign + * @return a FieldUpdate specifying the assignment + * @throws IllegalArgumentException if the runtime type of newValue does not match the type required by field + * @see com.yahoo.document.update.ValueUpdate#createAssign(FieldValue) + */ + public static FieldUpdate createAssign(Field field, FieldValue newValue) { + return new FieldUpdate(field, ValueUpdate.createAssign(newValue)); + } + + /** + * Creates a new field update, with one encapsulated value update, + * that clears the value; see documentation for ClearValueUpdate to see behavior + * for the individual data types. Note that clearing the value is not the same + * clearing the field; this method leaves an empty value, whereas clearing the + * field will completely remove the value. + * + * @param field the field to clear the value of + * @return a FieldUpdate specifying the clearing + * @see com.yahoo.document.update.ClearValueUpdate + * @see ValueUpdate#createClear() + * @see FieldUpdate#createClearField(com.yahoo.document.Field) + */ + public static FieldUpdate createClear(Field field) { + return new FieldUpdate(field, ValueUpdate.createClear()); + } + + /** + * Creates a new field update, with one encapsulated value update, which + * is able to map an update to a value to a subvalue in an array or a + * weighted set. If this update is to be applied to an array, the value parameter must be an integer specifying + * the index in the array that the update parameter is to be applied to, and the update parameter must be + * compatible with the sub-datatype of the array. If this update is to be applied on a weighted set, the value + * parameter must be the key in the set that the update parameter is to be applied to, and the update parameter + * must be compatible with the INT data type. + * + * @param field the field to modify the subvalue of + * @param value the index in case of array, or key in case of weighted set + * @param update the update to apply to the target sub-value + * @throws IllegalArgumentException in case data type is an array type and value is not an Integer; in case data type is a weighted set type and value is not equal to the nested type of the weighted set; or the encapsulated update throws such an exception + * @throws UnsupportedOperationException if superType is a single-value type, or anything else than array or weighted set; or the encapsulated update throws such an exception + * @return a FieldUpdate specifying the sub-update + * @see ValueUpdate#createMap(FieldValue, ValueUpdate) + */ + public static FieldUpdate createMap(Field field, FieldValue value, ValueUpdate update) { + return new FieldUpdate(field, ValueUpdate.createMap(value, update)); + } + + /** + * Creates a new field update, with one encapsulated value update, + * specifying the removal of a value from an array or a key/weight from a weighted set. + * + * @param field the field to remove a value from + * @param value the value to remove from the array, or key to remove from the weighted set + * @return a FieldUpdate specifying the removal + * @throws IllegalArgumentException if the runtime type of newValue does not match the type required + * @throws UnsupportedOperationException if the field type is not array or weighted set + * @see ValueUpdate#createRemove(FieldValue) + */ + public static FieldUpdate createRemove(Field field, FieldValue value) { + return new FieldUpdate(field, ValueUpdate.createRemove(value)); + } + + /** + * Creates a new field update, with encapsulated value updates, + * specifying the removal of all values in a given list from an array or weighted set. Note that this method + * is just a convenience method, it simply iterates + * through the list and creates value updates by calling createRemove() for each element. + * + * @param field the field to remove values from + * @param values a List containing the values to remove + * @return a FieldUpdate specifying the removal + * @throws IllegalArgumentException if the runtime type of values does not match the type required by field + * @throws UnsupportedOperationException if the field type is not array or weighted set + * @see ValueUpdate#createRemoveAll(java.util.List) + */ + public static FieldUpdate createRemoveAll(Field field, List<? extends FieldValue> values) { + return new FieldUpdate(field, ValueUpdate.createRemoveAll(values)); + } + + /** + * Creates a new field update, with encapsulated value updates, + * specifying the removal of all values in a given list from an array or weighted set. Note that this method + * is just a convenience method, it simply iterates + * through the list and creates value updates by calling createRemove() for each element. + * + * @param field the field to remove values from + * @param values a List containing the values to remove + * @return a FieldUpdate specifying the removal + * @throws IllegalArgumentException if the runtime type of values does not match the type required by field + * @throws UnsupportedOperationException if the field type is not array or weighted set + * @see ValueUpdate#createRemoveAll(java.util.List) + */ + public static FieldUpdate createRemoveAll(Field field, WeightedSet<? extends FieldValue> values) { + return new FieldUpdate(field, ValueUpdate.createRemoveAll(values)); + } +} diff --git a/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java b/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java new file mode 100644 index 00000000000..37b4329c934 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.update; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.StructuredDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.serialization.DocumentUpdateWriter; + +/** + * <p>Value update that represents performing an encapsulated value update on a subvalue. Currently, there are + * two multi-value data types in Vespa, <em>array</em> and <em>weighted set</em>. </p> + * + * <ul> + * <li>For an array, the value + * must be an Integer, and the update must represent a legal operation on the subtype of the array. </li> + * <li>For a + * weighted set, the value must be a key of the same type as the subtype of the weighted set, and the update + * must represent a legal operation on an integer value.</li> + * </ul> + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class MapValueUpdate extends ValueUpdate { + protected FieldValue value; + protected ValueUpdate update; + + public MapValueUpdate(FieldValue value, ValueUpdate update) { + super(ValueUpdateClassID.MAP); + this.value = value; + this.update = update; + } + + /** Returns the key of the nested update */ + public FieldValue getValue() { + return value; + } + + /** Sets the key of the nested update */ + public void setValue(FieldValue value) { + this.value=value; + } + + public ValueUpdate getUpdate() { + return update; + } + + @Override + public FieldValue applyTo(FieldValue fval) { + if (fval instanceof Array) { + Array array = (Array) fval; + FieldValue element = array.getFieldValue(((IntegerFieldValue) value).getInteger()); + element = update.applyTo(element); + array.set(((IntegerFieldValue) value).getInteger(), element); + } else if (fval instanceof WeightedSet) { + WeightedSet wset = (WeightedSet) fval; + WeightedSetDataType wtype = wset.getDataType(); + Integer weight = wset.get(value); + if (weight == null) { + if (wtype.createIfNonExistent() && update instanceof ArithmeticValueUpdate) { + weight = 0; + } else { + return fval; + } + } + weight = (Integer) update.applyTo(new IntegerFieldValue(weight)).getWrappedValue(); + wset.put((FieldValue) value, weight); + if (wtype.removeIfZero() && update instanceof ArithmeticValueUpdate && weight == 0) { + wset.remove(value); + } + } + return fval; + } + + @Override + protected void checkCompatibility(DataType fieldType) { + if (fieldType instanceof ArrayDataType) { + if (!(value instanceof IntegerFieldValue)) { + throw new IllegalArgumentException("Expected integer, got " + value.getClass().getName() + "."); + } + update.checkCompatibility(((ArrayDataType)fieldType).getNestedType()); + } else if (fieldType instanceof WeightedSetDataType) { + ((WeightedSetDataType)fieldType).getNestedType().createFieldValue().assign(value); + update.checkCompatibility(DataType.INT); + } else if (fieldType instanceof StructuredDataType) { + if (!(value instanceof StringFieldValue)) { + throw new IllegalArgumentException("Expected string, got " + value.getClass().getName() + "."); + } + Field field = ((StructuredDataType)fieldType).getField(((StringFieldValue)value).getString()); + if (field == null) { + throw new IllegalArgumentException("Field '" + value + "' not found."); + } + update.checkCompatibility(field.getDataType()); + } else { + throw new UnsupportedOperationException("Field type " + fieldType.getName() + " not supported."); + } + } + + + @Override + public void serialize(DocumentUpdateWriter data, DataType superType) { + data.write(this, superType); + } + + @Override + public boolean equals(Object o) { + return o instanceof MapValueUpdate && super.equals(o) && value.equals(((MapValueUpdate) o).value) && + update.equals(((MapValueUpdate) o).update); + } + + @Override + public int hashCode() { + return super.hashCode() + value.hashCode() + update.hashCode(); + } + + @Override + public String toString() { + return super.toString() + " " + value + " " + update; + } + +} + diff --git a/document/src/main/java/com/yahoo/document/update/RemoveValueUpdate.java b/document/src/main/java/com/yahoo/document/update/RemoveValueUpdate.java new file mode 100644 index 00000000000..e83d19803b1 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/update/RemoveValueUpdate.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.update; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.serialization.DocumentUpdateWriter; + +/** + * <p>Value update representing a removal of a value (and its associated weight, if any) + * from a multi-valued data type.</p> + * Deprecated: Use RemoveFieldPathUpdate instead. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class RemoveValueUpdate extends ValueUpdate { + protected FieldValue value; + + public RemoveValueUpdate(FieldValue value) { + super(ValueUpdateClassID.REMOVE); + this.value = value; + } + + /** Sets the key this should remove */ + public FieldValue getValue() { return value; } + + public void setValue(FieldValue value) { this.value=value; } + + @Override + public FieldValue applyTo(FieldValue fval) { + if (fval instanceof CollectionFieldValue) { + CollectionFieldValue val = (CollectionFieldValue) fval; + val.removeValue(value); + } + return fval; + } + + @Override + protected void checkCompatibility(DataType fieldType) { + if (!(fieldType instanceof CollectionDataType)) { + throw new UnsupportedOperationException("Expected collection, got " + fieldType.getName() + "."); + } + fieldType = ((CollectionDataType)fieldType).getNestedType(); + if (value != null && !value.getDataType().equals(fieldType)) { + throw new IllegalArgumentException("Expected " + fieldType.getName() + ", got " + + value.getDataType().getName()); + } + } + + @Override + public void serialize(DocumentUpdateWriter data, DataType superType) { + data.write(this, superType); + } + + @Override + public boolean equals(Object o) { + return o instanceof RemoveValueUpdate && super.equals(o) && value.equals(((RemoveValueUpdate) o).value); + } + + @Override + public int hashCode() { + return super.hashCode() + value.hashCode(); + } + + @Override + public String toString() { + return super.toString() + " " + value; + } +} diff --git a/document/src/main/java/com/yahoo/document/update/ValueUpdate.java b/document/src/main/java/com/yahoo/document/update/ValueUpdate.java new file mode 100644 index 00000000000..5c6f5b22dbe --- /dev/null +++ b/document/src/main/java/com/yahoo/document/update/ValueUpdate.java @@ -0,0 +1,364 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.update; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.serialization.DocumentUpdateWriter; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * A value update represents some action to perform to a value. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @see com.yahoo.document.update.FieldUpdate + * @see com.yahoo.document.DocumentUpdate + * @see AddValueUpdate + * @see com.yahoo.document.update.ArithmeticValueUpdate + * @see com.yahoo.document.update.AssignValueUpdate + * @see com.yahoo.document.update.ClearValueUpdate + * @see com.yahoo.document.update.MapValueUpdate + * @see com.yahoo.document.update.RemoveValueUpdate + */ +public abstract class ValueUpdate<T extends FieldValue> { + + protected ValueUpdateClassID valueUpdateClassID; + + protected ValueUpdate(ValueUpdateClassID valueUpdateClassID) { + this.valueUpdateClassID = valueUpdateClassID; + } + + /** + * Returns the valueUpdateClassID of this value update. + * + * @return the valueUpdateClassID of this ValueUpdate + */ + public ValueUpdateClassID getValueUpdateClassID() { + return valueUpdateClassID; + } + + protected abstract void checkCompatibility(DataType fieldType); + + public abstract void serialize(DocumentUpdateWriter data, DataType superType); + + @Override + public boolean equals(Object o) { + return o instanceof ValueUpdate && valueUpdateClassID == ((ValueUpdate) o).valueUpdateClassID; + } + + @Override + public int hashCode() { + return valueUpdateClassID.id; + } + + @Override + public String toString() { + return valueUpdateClassID.name; + } + + public abstract FieldValue applyTo(FieldValue oldValue); + + /** + * Creates a new value update specifying an addition of a value to an array or a key to a weighted set (with default weight 1). + * + * @param value the value to add to the array, or key to add to the weighted set + * @return a ValueUpdate specifying the addition + * @throws IllegalArgumentException if the runtime type of newValue does not match the type required + * @throws UnsupportedOperationException if the field type is not array or weighted set + */ + public static ValueUpdate createAdd(FieldValue value) { + return new AddValueUpdate(value); + } + + /** + * Creates a new value update specifying an addition of a key (with a specified weight) to a weighted set. If this + * method is used on an array data type, the weight will be omitted. + * + * @param key the key to add + * @param weight the weight to associate with the given key + * @return a ValueUpdate specifying the addition + * @throws IllegalArgumentException if the runtime type of key does not match the type required + * @throws UnsupportedOperationException if the field type is not array or weighted set + */ + public static ValueUpdate createAdd(FieldValue key, Integer weight) { + return new AddValueUpdate(key, weight); + } + + /** + * Creates a new value update + * specifying an addition of all values in a given list to an array. If this method is used on a weighted set data + * type, the default weights will be 1. Note that this method is just a convenience method, it simply iterates + * through the list and creates value updates by calling createAdd() for each element. + * + * @param values a List containing the values to add + * @return a List of ValueUpdates specifying the addition + * @throws IllegalArgumentException if the runtime type of values does not match the type required + * @throws UnsupportedOperationException if the field type is not array or weighted set + * @see ValueUpdate#createAdd(FieldValue) + */ + public static List<ValueUpdate> createAddAll(List<? extends FieldValue> values) { + List<ValueUpdate> vupds = new ArrayList<>(); + for (FieldValue value : values) { + vupds.add(ValueUpdate.createAdd(value)); + } + return vupds; + } + + /** + * Creates a new value update + * specifying an addition of all key/weight pairs in a weighted set to a weighted set. If this method + * is used on an array data type, the weights will be omitted. Note that this method is just a convenience method, + * it simply iterates through the set and creates value updates by calling createAdd() for each element. + * + * @param set a WeightedSet containing the key/weight pairs to add + * @return a ValueUpdate specifying the addition + * @throws IllegalArgumentException if the runtime type of values does not match the type required + * @throws UnsupportedOperationException if the field type is not weighted set or array + * @see ValueUpdate#createAdd(FieldValue, Integer) + */ + public static List<ValueUpdate> createAddAll(WeightedSet<? extends FieldValue> set) { + List<ValueUpdate> vupds = new ArrayList<>(); + Iterator<? extends FieldValue> it = set.fieldValueIterator(); + while (it.hasNext()) { + FieldValue key = it.next(); + vupds.add(ValueUpdate.createAdd(key, set.get(key))); + } + return vupds; + } + + /** + * Creates a new value update that increments a value. Note that the data type must be a numeric + * type. + * + * @param increment the number to increment by + * @return a ValueUpdate specifying the increment + * @throws UnsupportedOperationException if the data type is non-numeric + */ + public static ValueUpdate createIncrement(Number increment) { + return new ArithmeticValueUpdate(ArithmeticValueUpdate.Operator.ADD, increment); + } + + /** + * Creates a new value update that increments a weight in a weighted set. Note that this method is just a convenience + * method, it simply creates an increment value update by calling createIncrement() and then creates a map value + * update by calling createMap() with the key and the increment value update as parameters. + * + * @param key the key whose weight in the weighted set to increment + * @param increment the number to increment by + * @return a ValueUpdate specifying the increment + * @see ValueUpdate#createIncrement(Number) + * @see ValueUpdate#createMap(FieldValue, ValueUpdate) + */ + public static ValueUpdate createIncrement(FieldValue key, Number increment) { + return createMap(key, createIncrement(increment)); + } + + /** + * Creates a new value update that decrements a value. Note that the data type must be a numeric + * type. + * + * @param decrement the number to decrement by + * @return a ValueUpdate specifying the decrement + * @throws UnsupportedOperationException if the data type is non-numeric + */ + public static ValueUpdate createDecrement(Number decrement) { + return new ArithmeticValueUpdate(ArithmeticValueUpdate.Operator.SUB, decrement); + } + + /** + * Creates a new value update that decrements a weight in a weighted set. Note that this method is just a convenience + * method, it simply creates a decrement value update by calling createDecrement() and then creates a map value + * update by calling createMap() with the key and the decrement value update as parameters. + * + * @param key the key whose weight in the weighted set to decrement + * @param decrement the number to decrement by + * @return a ValueUpdate specifying the decrement + * @see ValueUpdate#createDecrement(Number) + * @see ValueUpdate#createMap(FieldValue, ValueUpdate) + */ + public static ValueUpdate createDecrement(FieldValue key, Number decrement) { + return createMap(key, createDecrement(decrement)); + } + + /** + * Creates a new value update that multiplies a value. Note that the data type must be a numeric + * type. + * + * @param factor the number to multiply by + * @return a ValueUpdate specifying the multiplication + * @throws UnsupportedOperationException if the data type is non-numeric + */ + public static ValueUpdate createMultiply(Number factor) { + return new ArithmeticValueUpdate(ArithmeticValueUpdate.Operator.MUL, factor); + } + + /** + * Creates a new value update that multiplies a weight in a weighted set. Note that this method is just a convenience + * method, it simply creates a multiply value update by calling createMultiply() and then creates a map value + * update by calling createMap() with the key and the multiply value update as parameters. + * + * @param key the key whose weight in the weighted set to multiply + * @param factor the number to multiply by + * @return a ValueUpdate specifying the multiplication + * @see ValueUpdate#createMultiply(Number) + * @see ValueUpdate#createMap(FieldValue, ValueUpdate) + */ + public static ValueUpdate createMultiply(FieldValue key, Number factor) { + return createMap(key, createMultiply(factor)); + } + + + /** + * Creates a new value update that divides a value. Note that the data type must be a numeric + * type. + * + * @param divisor the number to divide by + * @return a ValueUpdate specifying the division + * @throws UnsupportedOperationException if the data type is non-numeric + */ + public static ValueUpdate createDivide(Number divisor) { + return new ArithmeticValueUpdate(ArithmeticValueUpdate.Operator.DIV, divisor); + } + + /** + * Creates a new value update that divides a weight in a weighted set. Note that this method is just a convenience + * method, it simply creates a divide value update by calling createDivide() and then creates a map value + * update by calling createMap() with the key and the divide value update as parameters. + * + * @param key the key whose weight in the weighted set to divide + * @param divisor the number to divide by + * @return a ValueUpdate specifying the division + * @see ValueUpdate#createDivide(Number) + * @see ValueUpdate#createMap(FieldValue, ValueUpdate) + */ + public static ValueUpdate createDivide(FieldValue key, Number divisor) { + return createMap(key, createDivide(divisor)); + } + + /** + * Creates a new value update that assigns a new value, completely overwriting + * the previous value. + * + * @param newValue the value to assign + * @return a ValueUpdate specifying the assignment + * @throws IllegalArgumentException if the runtime type of newValue does not match the type required + */ + public static ValueUpdate createAssign(FieldValue newValue) { + return new AssignValueUpdate(newValue); + } + + /** + * Creates a new value update that clears the field fromthe document. + * + * @return a ValueUpdate specifying the removal + */ + public static ValueUpdate createClear() { + return new ClearValueUpdate(); + } + + /** + * Creates a map value update, which is able to map an update to a value to a subvalue in an array or a + * weighted set. If this update is to be applied to an array, the value parameter must be an integer specifying + * the index in the array that the update parameter is to be applied to, and the update parameter must be + * compatible with the sub-datatype of the array. If this update is to be applied on a weighted set, the value + * parameter must be the key in the set that the update parameter is to be applied to, and the update parameter + * must be compatible with the INT data type. + * + * @param value the index in case of array, or key in case of weighted set + * @param update the update to apply to the target sub-value + * @throws IllegalArgumentException in case data type is an array type and value is not an Integer; in case data type is a weighted set type and value is not equal to the nested type of the weighted set; or the encapsulated update throws such an exception + * @throws UnsupportedOperationException if superType is a single-value type, or anything else than array or weighted set; or the encapsulated update throws such an exception + * @return a ValueUpdate specifying the sub-update + */ + public static ValueUpdate createMap(FieldValue value, ValueUpdate update) { + return new MapValueUpdate(value, update); + } + + /** + * Creates a new value update specifying the removal of a value from an array or a key/weight from a weighted set. + * + * @param value the value to remove from the array, or key to remove from the weighted set + * @return a ValueUpdate specifying the removal + * @throws IllegalArgumentException if the runtime type of newValue does not match the type required + * @throws UnsupportedOperationException if the field type is not array or weighted set + */ + public static ValueUpdate createRemove(FieldValue value) { + return new RemoveValueUpdate(value); + } + + /** + * Creates a new value update + * specifying the removal of all values in a given list from an array or weighted set. Note that this method + * is just a convenience method, it simply iterates + * through the list and creates value updates by calling createRemove() for each element. + * + * @param values a List containing the values to remove + * @return a List of ValueUpdates specifying the removal + * @throws IllegalArgumentException if the runtime type of values does not match the type required + * @throws UnsupportedOperationException if the field type is not array or weighted set + * @see ValueUpdate#createRemove(FieldValue) + */ + public static List<ValueUpdate> createRemoveAll(List<? extends FieldValue> values) { + List<ValueUpdate> vupds = new ArrayList<>(); + for (FieldValue value : values) { + vupds.add(ValueUpdate.createRemove(value)); + } + return vupds; + } + + /** + * Creates a new value update + * specifying the removal of all values in a given list from an array or weighted set. Note that this method + * is just a convenience method, it simply iterates + * through the list and creates value updates by calling createRemove() for each element. + * + * @param values a List containing the values to remove + * @return a List of ValueUpdates specifying the removal + * @throws IllegalArgumentException if the runtime type of values does not match the type required + * @throws UnsupportedOperationException if the field type is not array or weighted set + * @see ValueUpdate#createRemove(FieldValue) + */ + public static List<ValueUpdate> createRemoveAll(WeightedSet<? extends FieldValue> values) { + List<ValueUpdate> vupds = new ArrayList<>(); + for (FieldValue value : values.keySet()) { + vupds.add(ValueUpdate.createRemove(value)); + } + return vupds; + } + + /** Returns the primary "value" of this update, or null if this kind of update has no value */ + public abstract T getValue(); + + /** Sets the value of this. Ignored by update who have no value */ + public abstract void setValue(T value); + + public enum ValueUpdateClassID { + //DO NOT change anything here unless you change src/vespa/document/util/identifiableid.h as well!! + ADD(25, "add"), + ARITHMETIC(26, "arithmetic"), + ASSIGN(27, "assign"), + CLEAR(28, "clear"), + MAP(29, "map"), + REMOVE(30, "remove"); + + public final int id; + public final String name; + + ValueUpdateClassID(int id, String name) { + this.id = 0x1000 + id; + this.name = name; + } + + public static ValueUpdateClassID getID(int id) { + for (ValueUpdateClassID vucid : ValueUpdateClassID.values()) { + if (vucid.id == id) { + return vucid; + } + } + return null; + } + } +} diff --git a/document/src/main/java/com/yahoo/document/update/package-info.java b/document/src/main/java/com/yahoo/document/update/package-info.java new file mode 100644 index 00000000000..3699dec32ef --- /dev/null +++ b/document/src/main/java/com/yahoo/document/update/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.document.update; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/documentmodel/.gitignore b/document/src/main/java/com/yahoo/documentmodel/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/document/src/main/java/com/yahoo/documentmodel/.gitignore diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java new file mode 100644 index 00000000000..e97bd43d9bf --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; + +/** + * Minimal interface for reading operations from a stream for a feeder. + * + * Interface extracted from VespaXMLFeedReader to enable JSON feeding. + * + * @author steinar + */ +public interface FeedReader { + + /** + * Reads the next operation from the stream. + * @param operation The operation to fill in. Operation is unchanged if none was found. + */ + public abstract void read(Operation operation) throws Exception; + +}
\ No newline at end of file diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLDocumentReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLDocumentReader.java new file mode 100644 index 00000000000..a5ea5983e29 --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLDocumentReader.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.serialization.DocumentReader; + +import javax.xml.stream.XMLStreamReader; +import java.io.InputStream; + +/** + * XML parser that reads Vespa documents from an XML stream. + * + * @author thomasg + */ +public class VespaXMLDocumentReader extends VespaXMLFieldReader implements DocumentReader { + + /** + * Creates a reader that reads from the given file. + */ + public VespaXMLDocumentReader(String fileName, DocumentTypeManager docTypeManager) throws Exception { + super(fileName, docTypeManager); + } + + /** + * Creates a reader that reads from the given stream. + */ + public VespaXMLDocumentReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception { + super(stream, docTypeManager); + } + + /** + * Creates a reader that reads using the given reader. This is useful if the document is part of a greater + * XML stream. + */ + public VespaXMLDocumentReader(XMLStreamReader reader, DocumentTypeManager docTypeManager) { + super(reader, docTypeManager); + } + + /** + * Reads one document from the stream. Function assumes that the current element in the stream is + * the start tag for the document. + * + * @param document the document to be read + */ + public void read(Document document) { + read(null, document); + } +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java new file mode 100644 index 00000000000..0c8b9b22961 --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java @@ -0,0 +1,313 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +/** + * XML parser for Vespa document XML. + * + * Parses an entire document "feed", which consists of a vespafeed element containing + * zero or more instances of documents, updates or removes. + * + * Standard usage is to create an Operation object and call read(Operation) until + * operation.getType() returns OperationType.INVALID. + * + * If you are looking to parse only a single document or update, use VespaXMLDocumentReader + * or VespaXMLUpdateReader respectively. + */ +public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { + + /** + * Creates a reader that reads from the given file. + */ + public VespaXMLFeedReader(String fileName, DocumentTypeManager docTypeManager) throws Exception { + super(fileName, docTypeManager); + readInitial(); + } + + /** + * Creates a reader that reads from the given stream. + */ + public VespaXMLFeedReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception { + super(stream, docTypeManager); + readInitial(); + } + + /** + * Creates a reader that uses the given reader to read - this can be used if the vespa feed + * is part of a larger XML document. + */ + public VespaXMLFeedReader(XMLStreamReader reader, DocumentTypeManager manager) throws Exception { + super(reader, manager); + readInitial(); + } + + /** + * Skips the initial "vespafeed" tag. + */ + void readInitial() throws Exception { + boolean found = false; + + while (reader.hasNext()) { + int type = reader.next(); + if (type == XMLStreamReader.START_ELEMENT) { + if ("vespafeed".equals(reader.getName().toString())) { + found = true; + break; + } + } + } + + if (!found) { + throw newDeserializeException("Feed information must be contained within a \"vespafeed\" element"); + } + } + + public enum OperationType { + DOCUMENT, + REMOVE, + UPDATE, + INVALID + } + + /** + * Represents a feed operation found by the parser. Can be one of the following types: + * - getType() == DOCUMENT: getDocument() is valid. + * - getType() == REMOVE: getRemove() is valid. + * - getType() == UPDATE: getUpdate() is valid. + */ + public static class Operation { + + private OperationType type; + private Document doc; + private DocumentId remove; + private DocumentUpdate docUpdate; + private FeedOperation feedOperation; + private TestAndSetCondition condition; + + public Operation() { + setInvalid(); + } + + public void setInvalid() { + type = OperationType.INVALID; + doc = null; + remove = null; + docUpdate = null; + feedOperation = null; + condition = null; + } + + public OperationType getType() { + return type; + } + + public Document getDocument() { + return doc; + } + + public void setDocument(Document doc) { + this.type = OperationType.DOCUMENT; + this.doc = doc; + } + + public DocumentId getRemove() { + return remove; + } + + public void setRemove(DocumentId remove) { + this.type = OperationType.REMOVE; + this.remove = remove; + } + + public DocumentUpdate getDocumentUpdate() { + return docUpdate; + } + + public void setDocumentUpdate(DocumentUpdate docUpdate) { + this.type = OperationType.UPDATE; + this.docUpdate = docUpdate; + } + + public FeedOperation getFeedOperation() { + return feedOperation; + } + + public void setCondition(TestAndSetCondition condition) { + this.condition = condition; + } + + public TestAndSetCondition getCondition() { + return condition; + } + + @Override + public String toString() { + return "Operation{" + + "type=" + type + + ", doc=" + doc + + ", remove=" + remove + + ", docUpdate=" + docUpdate + + ", feedOperation=" + feedOperation + + '}'; + } + } + + public static class FeedOperation { + + private String name; + private Integer generation; + private Integer increment; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getGeneration() { + return generation; + } + + public void setGeneration(int generation) { + this.generation = generation; + } + + public Integer getIncrement() { + return increment; + } + + public void setIncrement(int increment) { + this.increment = increment; + } + } + + /** + * <p>Reads all operations from the XML stream and puts into a list. Note + * that if the XML stream is large, this may cause out of memory errors, so + * make sure to use this only with small streams.</p> + * + * @return The list of all read operations. + */ + public List<Operation> readAll() throws Exception { + List<Operation> list = new ArrayList<Operation>(); + while (true) { + Operation op = new Operation(); + read(op); + if (op.getType() == OperationType.INVALID) { + return list; + } else { + list.add(op); + } + } + } + + /* (non-Javadoc) + * @see com.yahoo.vespaxmlparser.FeedReader#read(com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation) + */ + @Override + public void read(Operation operation) throws Exception { + String startTag = null; + operation.setInvalid(); + + try { + while (reader.hasNext()) { + int type = reader.next(); + + if (type == XMLStreamReader.START_ELEMENT) { + startTag = reader.getName().toString(); + + if ("document".equals(startTag)) { + VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(reader, docTypeManager); + Document document = new Document(documentReader); + operation.setDocument(document); + operation.setCondition(TestAndSetCondition.fromConditionString(documentReader.getCondition())); + return; + } else if ("update".equals(startTag)) { + VespaXMLUpdateReader updateReader = new VespaXMLUpdateReader(reader, docTypeManager); + DocumentUpdate update = new DocumentUpdate(updateReader); + operation.setDocumentUpdate(update); + operation.setCondition(TestAndSetCondition.fromConditionString(updateReader.getCondition())); + return; + } else if ("remove".equals(startTag)) { + boolean documentIdFound = false; + + Optional<String> condition = Optional.empty(); + for (int i = 0; i < reader.getAttributeCount(); i++) { + final String attributeName = reader.getAttributeName(i).toString(); + if ("documentid".equals(attributeName) || "id".equals(attributeName)) { + operation.setRemove(new DocumentId(reader.getAttributeValue(i))); + documentIdFound = true; + } else if ("condition".equals(attributeName)) { + condition = Optional.of(reader.getAttributeValue(i)); + } + } + + if (!documentIdFound) { + throw newDeserializeException("Missing \"documentid\" attribute for remove operation"); + } + + operation.setCondition(TestAndSetCondition.fromConditionString(condition)); + + return; + } else { + throw newDeserializeException("Element \"" + startTag + "\" not allowed in this context"); + } + } + } + } catch (XMLStreamException e) { + throw(e); + // Skip to end of current tag with other exceptions. + } catch (Exception e) { + try { + if (startTag != null) { + skipToEnd(startTag); + } + } catch (Exception ignore) { + } + + throw(e); + } + } + + public void read(FeedOperation fo) throws XMLStreamException { + while (reader.hasNext()) { + int type = reader.next(); + + if (type == XMLStreamReader.START_ELEMENT) { + if ("name".equals(reader.getName().toString())) { + fo.setName(reader.getElementText().toString()); + skipToEnd("name"); + } else if ("generation".equals(reader.getName().toString())) { + fo.setGeneration(Integer.parseInt(reader.getElementText().toString())); + skipToEnd("generation"); + } else if ("increment".equals(reader.getName().toString())) { + String text = reader.getElementText(); + if ("autodetect".equals(text)) { + fo.setIncrement(-1); + } else { + fo.setIncrement(Integer.parseInt(text)); + } + skipToEnd("increment"); + } + } else if (type == XMLStreamReader.END_ELEMENT) { + return; + } + } + } + +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java new file mode 100644 index 00000000000..cdc676eca5f --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFieldReader.java @@ -0,0 +1,520 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.datatypes.*; +import com.yahoo.document.predicate.Predicate; +import com.yahoo.document.serialization.DeserializationException; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.objects.FieldBase; +import org.apache.commons.codec.binary.Base64; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import java.io.InputStream; +import java.math.BigInteger; +import java.util.Optional; + +/** + * XML parser that reads document fields from an XML stream. + * + * All read methods assume that the stream is currently positioned at the start element of the relevant field. + * + */ +public class VespaXMLFieldReader extends VespaXMLReader implements FieldReader { + private static final BigInteger UINT_MAX = new BigInteger("4294967296"); + private static final BigInteger ULONG_MAX = new BigInteger("18446744073709551616"); + + public VespaXMLFieldReader(String fileName, DocumentTypeManager docTypeManager) throws Exception { + super(fileName, docTypeManager); + } + + public VespaXMLFieldReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception { + super(stream, docTypeManager); + } + + public VespaXMLFieldReader(XMLStreamReader reader, DocumentTypeManager docTypeManager) { + super(reader, docTypeManager); + } + + /** + * Optional test and set condition. Common for document/update/remove elements + * This variable is either set in VespaXMLFieldReader#read (reader for document) + * or in VespaXMLUpdateReader#read (reader for update). + */ + private Optional<String> condition = Optional.empty(); + + public Optional<String> getCondition() { + return condition; + } + + public void read(FieldBase field, Document document) { + try { + //workaround for documents inside array <item> + if (reader.getEventType() != XMLStreamReader.START_ELEMENT || !"document".equals(reader.getName().toString())) { + while (reader.hasNext()) { + if (reader.getEventType() == XMLStreamReader.START_ELEMENT && "document".equals(reader.getName().toString())) { + break; + } + reader.next(); + } + } + + // First fetch attributes. + String typeName = null; + + for (int i = 0; i < reader.getAttributeCount(); i++) { + final String attributeName = reader.getAttributeName(i).toString(); + if ("documentid".equals(attributeName) || "id".equals(attributeName)) { + document.setId(new DocumentId(reader.getAttributeValue(i))); + } else if ("documenttype".equals(attributeName) || "type".equals(attributeName)) { + typeName = reader.getAttributeValue(i); + } else if ("condition".equals(attributeName)) { + condition = Optional.of(reader.getAttributeValue(i)); + } + } + + if (document.getId() != null) { + if (field == null) { + field = new FieldBase(document.getId().toString()); + } + } + + DocumentType doctype = docTypeManager.getDocumentType(typeName); + if (doctype == null) { + throw newDeserializeException(field, "Must specify an existing document type, not '" + typeName + "'"); + } else { + document.setDataType(doctype); + } + + // Then fetch fields + while (reader.hasNext()) { + int type = reader.next(); + + if (type == XMLStreamReader.START_ELEMENT) { + Field f = doctype.getField(reader.getName().toString()); + + if (f == null) { + throw newDeserializeException(field, "Field " + reader.getName() + " not found."); + } + + FieldValue fv = f.getDataType().createFieldValue(); + fv.deserialize(f, this); + document.setFieldValue(f, fv); + skipToEnd(f.getName()); + } else if (type == XMLStreamReader.END_ELEMENT) { + return; + } + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + public <T extends FieldValue> void read(FieldBase field, Array<T> value) { + try { + while (reader.hasNext()) { + int type = reader.next(); + + if (type == XMLStreamReader.START_ELEMENT) { + if ("item".equals(reader.getName().toString())) { + FieldValue fv = (value.getDataType()).getNestedType().createFieldValue(); + deserializeFieldValue(field, fv); + // noinspection unchecked + value.add((T)fv); + skipToEnd("item"); + } + } else if (type == XMLStreamReader.END_ELEMENT) { + return; + } + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + class KeyAndValue { + FieldValue key = null; + FieldValue value = null; + } + + void readKeyAndValue(FieldBase field, KeyAndValue val, MapDataType dt) throws XMLStreamException { + while (reader.hasNext()) { + int type = reader.next(); + + if (type == XMLStreamReader.START_ELEMENT) { + if ("key".equals(reader.getName().toString())) { + val.key = dt.getKeyType().createFieldValue(); + deserializeFieldValue(field, val.key); + skipToEnd("key"); + } else if ("value".equals(reader.getName().toString())) { + val.value = dt.getValueType().createFieldValue(); + deserializeFieldValue(field, val.value); + skipToEnd("value"); + } else { + throw newDeserializeException("Illegal element inside map item: " + reader.getName()); + } + } else if (type == XMLStreamReader.END_ELEMENT) { + return; + } + } + } + + public <K extends FieldValue, V extends FieldValue> void read(FieldBase field, MapFieldValue<K, V> map) { + try { + MapDataType dt = map.getDataType(); + + while (reader.hasNext()) { + int type = reader.next(); + + if (type == XMLStreamReader.START_ELEMENT) { + if ("item".equals(reader.getName().toString())) { + KeyAndValue kv = new KeyAndValue(); + readKeyAndValue(field, kv, dt); + + if (kv.key == null || kv.value == null) { + throw newDeserializeException(field, "Map items must specify both key and value"); + } + // noinspection unchecked + map.put((K)kv.key, (V)kv.value); + skipToEnd("item"); + } else { + throw newDeserializeException(field, "Illegal tag " + reader.getName() + " expected 'item'"); + } + } else if (type == XMLStreamReader.END_ELEMENT) { + return; + } + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + public void read(FieldBase field, Struct value) { + try { + boolean base64 = isBase64EncodedElement(reader); + boolean foundField = false; + StringBuilder positionBuilder = null; + while (reader.hasNext()) { + int type = reader.next(); + if (type == XMLStreamReader.START_ELEMENT) { + Field structField = value.getField(reader.getName().toString()); + if (structField == null) { + throw newDeserializeException(field, "Field " + reader.getName() + " not found."); + } + FieldValue fieldValue = structField.getDataType().createFieldValue(); + fieldValue.deserialize(structField, this); + value.setFieldValue(structField, fieldValue); + skipToEnd(structField.getName()); + foundField = true; + } else if (type == XMLStreamReader.CHARACTERS) { + if (foundField) { + continue; + } + // The text of an XML element may be output using 1-n CHARACTERS + // events, so we have to buffer up until the end of the element to + // ensure we get everything. + String chars = reader.getText(); + if (positionBuilder == null) { + positionBuilder = new StringBuilder(chars); + } else { + positionBuilder.append(chars); + } + } else if (type == XMLStreamReader.END_ELEMENT) { + if (positionBuilder != null) { + assignPositionFieldFromStringIfNonEmpty(value, positionBuilder.toString(), base64); + } + break; + } + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + private void assignPositionFieldFromStringIfNonEmpty(Struct value, String elementText, boolean base64) { + String str = base64 ? Utf8.toString(new Base64().decode(elementText)) : elementText; + str = str.trim(); + if (str.isEmpty()) { + return; + } + DataType valueType = value.getDataType(); + if (valueType.equals(PositionDataType.INSTANCE)) { + value.assign(PositionDataType.fromString(str)); + } + } + + public <T extends FieldValue> void read(FieldBase field, WeightedSet<T> value) { + try { + while (reader.hasNext()) { + int type = reader.next(); + + if (type == XMLStreamReader.START_ELEMENT) { + if ("item".equals(reader.getName().toString())) { + FieldValue fv = value.getDataType().getNestedType().createFieldValue(); + + int weight = 1; + for (int i = 0; i < reader.getAttributeCount(); i++) { + if ("weight".equals(reader.getAttributeName(i).toString())) { + weight = Integer.parseInt(reader.getAttributeValue(i)); + } + } + + deserializeFieldValue(field, fv); + // noinspection unchecked + value.put((T)fv, weight); + skipToEnd("item"); + } else { + throw newDeserializeException(field, "Illegal tag " + reader.getName() + " expected 'item'"); + } + } else if (type == XMLStreamReader.END_ELEMENT) { + return; + } + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + public void read(FieldBase field, ByteFieldValue value) { + try { + String dataParsed = reader.getElementText(); + try { + value.assign(new Byte(dataParsed)); + } catch (Exception e) { + throw newDeserializeException(field, "Invalid byte \"" + dataParsed + "\"."); + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + public void read(FieldBase field, DoubleFieldValue value) { + try { + String dataParsed = reader.getElementText(); + try { + value.assign(new Double(dataParsed)); + } catch (Exception e) { + throw newDeserializeException(field, "Invalid double \"" + dataParsed + "\"."); + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + public void read(FieldBase field, FloatFieldValue value) { + try { + String dataParsed = reader.getElementText(); + try { + value.assign(new Float(dataParsed)); + } catch (Exception e) { + throw newDeserializeException(field, "Invalid float \"" + dataParsed + "\"."); + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + private RuntimeException newDeserializeException(FieldBase field, String msg) { + return newDeserializeException("Field '" + ((field == null) ? "null" : field.getName()) + "': " + msg); + } + private RuntimeException newException(FieldBase field, Exception e) { + return newDeserializeException("Field '" + ((field == null) ? "null" : field.getName()) + "': " + e.getMessage()); + } + public void read(FieldBase field, IntegerFieldValue value) { + try { + String dataParsed = reader.getElementText(); + + BigInteger val; + try { + if (dataParsed.startsWith("0x")) { + val = new BigInteger(dataParsed.substring(2), 16); + } else if (dataParsed.startsWith("0") && dataParsed.length() > 1) { + val = new BigInteger(dataParsed.substring(1), 8); + } else { + val = new BigInteger(dataParsed); + } + } catch (Exception e) { + throw newDeserializeException(field, "Invalid integer \"" + dataParsed + "\"."); + } + if (val.bitLength() > 32) { + throw newDeserializeException(field, "Invalid integer \"" + dataParsed + "\". Out of range."); + } + if (val.bitLength() == 32) { + if (val.compareTo(BigInteger.ZERO) == 1) { + // Flip to negative + val = val.subtract(UINT_MAX); + } else { + throw newDeserializeException(field, "Invalid integer \"" + dataParsed + "\". Out of range."); + } + } + + value.assign(val.intValue()); + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + public void read(FieldBase field, LongFieldValue value) { + try { + String dataParsed = reader.getElementText(); + + BigInteger val; + try { + if (dataParsed.startsWith("0x")) { + val = new BigInteger(dataParsed.substring(2), 16); + } else if (dataParsed.startsWith("0") && dataParsed.length() > 1) { + val = new BigInteger(dataParsed.substring(1), 8); + } else { + val = new BigInteger(dataParsed); + } + } catch (Exception e) { + throw newDeserializeException(field, "Invalid long \"" + dataParsed + "\"."); + } + if (val.bitLength() > 64) { + throw newDeserializeException(field, "Invalid long \"" + dataParsed + "\". Out of range."); + } + if (val.compareTo(BigInteger.ZERO) == 1 && val.bitLength() == 64) { + // Flip to negative + val = val.subtract(ULONG_MAX); + } + value.assign(val.longValue()); + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + public void read(FieldBase field, Raw value) { + try { + if (isBase64EncodedElement(reader)) { + value.assign(new Base64().decode(reader.getElementText())); + } else { + value.assign(reader.getElementText().getBytes()); + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + @Override + public void read(FieldBase field, PredicateFieldValue value) { + try { + if (isBase64EncodedElement(reader)) { + value.assign(Predicate.fromBinary(new Base64().decode(reader.getElementText()))); + } else { + value.assign(Predicate.fromString(reader.getElementText())); + } + } catch (XMLStreamException e) { + throw newException(field, e); + } + } + + public void read(FieldBase field, StringFieldValue value) { + try { + if (isBase64EncodedElement(reader)) { + throw new IllegalArgumentException("Attribute binaryencoding=base64 is not allowed for fields of type 'string'. To represent binary data, use type 'raw'."); + } else { + value.assign(reader.getElementText()); + } + } catch (XMLStreamException | IllegalArgumentException e) { + throw newException(field, e); + } + } + + @Override + public void read(FieldBase field, TensorFieldValue value) { + throw new DeserializationException("Field '"+ (field != null ? field.getName() : "null") + "': " + + "XML input for fields of type TENSOR is not supported. Please use JSON input instead."); + } + + public void read(FieldBase field, AnnotationReference value) { + System.out.println("Annotation value read!"); + } + + private void deserializeFieldValue(FieldBase field, FieldValue value) { + value.deserialize(field instanceof Field ? (Field)field : null, this); + } + + /***********************************************************************/ + /* UNUSED METHODS */ + /***********************************************************************/ + + @SuppressWarnings("UnusedDeclaration") + public DocumentId readDocumentId() { + return null; + } + + @SuppressWarnings("UnusedDeclaration") + public DocumentType readDocumentType() { + return null; //To change body of implemented methods use File | Settings | File Templates. + } + + @SuppressWarnings("UnusedDeclaration") + public DocumentTypeManager getDocumentTypeManager() { + return docTypeManager; + } + + @Override + public <T extends FieldValue> void read(FieldBase field, CollectionFieldValue<T> value) { + System.out.println("Should not be called!!!"); + } + + @Override + public void read(FieldBase field, StructuredFieldValue value) { + System.out.println("Should not be called!!!"); + } + + @Override + public void read(FieldBase field, FieldValue value) { + System.out.println("SHOULD NEVER BE CALLED? " + field.toString()); + } + + @Override + public byte getByte(FieldBase fieldBase) { + return 0; + } + + @Override + public short getShort(FieldBase fieldBase) { + return 0; + } + + @Override + public int getInt(FieldBase fieldBase) { + return 0; + } + + @Override + public long getLong(FieldBase fieldBase) { + return 0; + } + + @Override + public float getFloat(FieldBase fieldBase) { + return 0; + } + + @Override + public double getDouble(FieldBase fieldBase) { + return 0; + } + + @Override + public byte[] getBytes(FieldBase fieldBase, int i) { + return new byte[0]; + } + + @Override + public String getString(FieldBase fieldBase) { + return null; + } +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLReader.java new file mode 100644 index 00000000000..10c3676a965 --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLReader.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.serialization.DeserializationException; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import java.io.FileInputStream; +import java.io.InputStream; + +/** + * @author thomasg + */ +public class VespaXMLReader { + DocumentTypeManager docTypeManager; + XMLStreamReader reader; + + public VespaXMLReader(String fileName, DocumentTypeManager docTypeManager) throws Exception { + this(new FileInputStream(fileName), docTypeManager); + } + + public VespaXMLReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception { + this.docTypeManager = docTypeManager; + XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); + xmlInputFactory.setProperty("javax.xml.stream.isSupportingExternalEntities", Boolean.FALSE); + reader = xmlInputFactory.createXMLStreamReader(stream); + } + + public VespaXMLReader(XMLStreamReader reader, DocumentTypeManager docTypeManager) { + this.docTypeManager = docTypeManager; + this.reader = reader; + } + + protected RuntimeException newDeserializeException(String message) { + return new DeserializationException(message + " (at line " + reader.getLocation().getLineNumber() + ", column " + reader.getLocation().getColumnNumber() + ")"); + } + + protected RuntimeException newException(Exception e) { + return new DeserializationException(e.getMessage() + " (at line " + reader.getLocation().getLineNumber() + ", column " + reader.getLocation().getColumnNumber() + ")", e); + } + + protected void skipToEnd(String tagName) throws XMLStreamException { + while (reader.hasNext()) { + if (reader.getEventType() == XMLStreamReader.END_ELEMENT && tagName.equals(reader.getName().toString())) { + return; + } + reader.next(); + } + throw new DeserializationException("Missing end tag for element '" + tagName + "'" + reader.getLocation()); + } + + public static boolean isBase64EncodingAttribute(String attributeName, String attributeValue) { + return "binaryencoding".equals(attributeName) && + "base64".equalsIgnoreCase(attributeValue); + } + + public static boolean isBase64EncodedElement(XMLStreamReader reader) { + for (int i = 0; i < reader.getAttributeCount(); i++) { + if (isBase64EncodingAttribute(reader.getAttributeName(i).toString(), + reader.getAttributeValue(i))) + { + return true; + } + } + return false; + } +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java new file mode 100644 index 00000000000..a4d334848d5 --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLUpdateReader.java @@ -0,0 +1,379 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.serialization.DocumentUpdateReader; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.ValueUpdate; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import java.io.InputStream; +import java.util.List; +import java.util.Optional; + +public class VespaXMLUpdateReader extends VespaXMLFieldReader implements DocumentUpdateReader { + public VespaXMLUpdateReader(String fileName, DocumentTypeManager docTypeManager) throws Exception { + super(fileName, docTypeManager); + } + + public VespaXMLUpdateReader(InputStream stream, DocumentTypeManager docTypeManager) throws Exception { + super(stream, docTypeManager); + } + + public VespaXMLUpdateReader(XMLStreamReader reader, DocumentTypeManager docTypeManager) { + super(reader, docTypeManager); + } + + private Optional<String> condition = Optional.empty(); + + public Optional<String> getCondition() { + return condition; + } + + public boolean hasFieldPath() { + for (int i = 0; i < reader.getAttributeCount(); i++) { + if (reader.getAttributeName(i).toString().equals("fieldpath")) { + return true; + } + } + + return false; + } + + public void read(DocumentUpdate update) { + try { + // First fetch attributes. + DocumentType doctype = null; + + for (int i = 0; i < reader.getAttributeCount(); i++) { + final String attributeName = reader.getAttributeName(i).toString(); + final String attributeValue = reader.getAttributeValue(i); + + if ("documentid".equals(attributeName) || "id".equals(attributeName)) { + update.setId(new DocumentId(attributeValue)); + } else if ("documenttype".equals(attributeName) || "type".equals(attributeName)) { + doctype = docTypeManager.getDocumentType(attributeValue); + update.setDocumentType(doctype); + } else if ("create-if-non-existent".equals(attributeName)) { + if ("true".equals(attributeValue)) { + update.setCreateIfNonExistent(true); + } else if ("false".equals(attributeValue)) { + update.setCreateIfNonExistent(false); + } else { + throw newDeserializeException("'create-if-non-existent' must be either 'true' or 'false', was '" + attributeValue +"'"); + } + } else if ("condition".equals(attributeName)) { + condition = Optional.of(attributeValue); + } + } + + if (doctype == null) { + throw newDeserializeException("Must specify document type. " + reader.getLocation()); + } + + // Then fetch fields + while (reader.hasNext()) { + int type = reader.next(); + + if (type == XMLStreamReader.START_ELEMENT) { + final String currentName = reader.getName().toString(); + if (hasFieldPath()) { + if ("assign".equals(currentName)) { + update.addFieldPathUpdate(new AssignFieldPathUpdate(doctype, this)); + skipToEnd("assign"); + } else if ("add".equals(currentName)) { + update.addFieldPathUpdate(new AddFieldPathUpdate(doctype, this)); + skipToEnd("add"); + } else if ("remove".equals(currentName)) { + update.addFieldPathUpdate(new RemoveFieldPathUpdate(doctype, this)); + skipToEnd("remove"); + } else { + throw newDeserializeException("Unknown field path update operation " + reader.getName()); + } + } else { + if ("assign".equals(currentName)) { + update.addFieldUpdate(readAssign(update)); + skipToEnd("assign"); + } else if ("add".equals(currentName)) { + update.addFieldUpdate(readAdd(update)); + skipToEnd("add"); + } else if ("remove".equals(currentName)) { + update.addFieldUpdate(readRemove(update)); + skipToEnd("remove"); + } else if ("alter".equals(currentName)) { + update.addFieldUpdate(readAlter(update)); + skipToEnd("alter"); + } else if ("increment".equals(currentName) || + "decrement".equals(currentName) || + "multiply".equals(currentName) || + "divide".equals(currentName)) { + update.addFieldUpdate(readArithmeticField(update, currentName)); + skipToEnd(currentName); + } else { + throw newDeserializeException("Unknown update operation " + reader.getName()); + } + } + } else if (type == XMLStreamReader.END_ELEMENT) { + return; + } + } + } catch (XMLStreamException e) { + throw newException(e); + } + } + + FieldUpdate readAdd(DocumentUpdate update) throws XMLStreamException { + for (int i = 0; i < reader.getAttributeCount(); i++) { + if ("field".equals(reader.getAttributeName(i).toString())) { + Field f = update.getDocumentType().getField(reader.getAttributeValue(i)); + + FieldValue value = f.getDataType().createFieldValue(); + value.deserialize(f, this); + + if (value instanceof Array) { + List<FieldValue> l = ((Array)value).getValues(); + return FieldUpdate.createAddAll(f, l); + } else if (value instanceof WeightedSet) { + return FieldUpdate.createAddAll(f, ((WeightedSet) value)); + } else { + throw newDeserializeException("Add operation only applicable to multivalue lists"); + } + + } + } + throw newDeserializeException("Add update without field attribute"); + } + + + FieldUpdate readRemove(DocumentUpdate update) throws XMLStreamException { + for (int i = 0; i < reader.getAttributeCount(); i++) { + if ("field".equals(reader.getAttributeName(i).toString())) { + Field f = update.getDocumentType().getField(reader.getAttributeValue(i)); + + FieldValue value = f.getDataType().createFieldValue(); + value.deserialize(f, this); + + if (value instanceof Array) { + List<FieldValue> l = ((Array)value).getValues(); + return FieldUpdate.createRemoveAll(f, l); + } else if (value instanceof WeightedSet) { + return FieldUpdate.createRemoveAll(f, ((WeightedSet)value)); + } else { + throw newDeserializeException("Remove operation only applicable to multivalue lists"); + } + + } + } + throw newDeserializeException("Remove update without field attribute"); + } + + FieldUpdate readAssign(DocumentUpdate update) throws XMLStreamException { + for (int i = 0; i < reader.getAttributeCount(); i++) { + if ("field".equals(reader.getAttributeName(i).toString())) { + Field f = update.getDocumentType().getField(reader.getAttributeValue(i)); + + if (f == null) { + throw newDeserializeException("Field " + reader.getAttributeValue(i) + " not found."); + } + + FieldValue value = f.getDataType().createFieldValue(); + value.deserialize(f, this); + return FieldUpdate.createAssign(f, value); + } + } + throw newDeserializeException("Assignment update without field attribute"); + } + + + FieldUpdate readAlter(DocumentUpdate update) throws XMLStreamException { + Field f = null; + for (int i = 0; i < reader.getAttributeCount(); i++) { + if ("field".equals(reader.getAttributeName(i).toString())) { + f = update.getDocumentType().getField(reader.getAttributeValue(i)); + } + } + + if (f == null) { + throw newDeserializeException("Alter update without \"field\" attribute"); + } + + FieldUpdate fu = FieldUpdate.create(f); + + while (reader.hasNext()) { + int type = reader.next(); + if (type == XMLStreamReader.START_ELEMENT) { + if ("increment".equals(reader.getName().toString()) || + "decrement".equals(reader.getName().toString()) || + "multiply".equals(reader.getName().toString()) || + "divide".equals(reader.getName().toString())) { + update.addFieldUpdate(readArithmetic(update, reader.getName().toString(), f, fu)); + skipToEnd(reader.getName().toString()); + } else { + throw newDeserializeException("Element \"" + reader.getName() + "\" not appropriate within alter element"); + } + } else if (type == XMLStreamReader.END_ELEMENT) { + break; + } + } + + return fu; + } + + FieldUpdate readArithmeticField(DocumentUpdate update, String type) throws XMLStreamException { + Field f = null; + for (int i = 0; i < reader.getAttributeCount(); i++) { + if ("field".equals(reader.getAttributeName(i).toString())) { + f = update.getDocumentType().getField(reader.getAttributeValue(i)); + } + } + + if (f == null) { + throw newDeserializeException("Assignment update without \"field\" attribute"); + } + + FieldUpdate fu = FieldUpdate.create(f); + readArithmetic(update, type, f, fu); + return fu; + } + + FieldUpdate readArithmetic(DocumentUpdate update, String type, Field f, FieldUpdate fu) throws XMLStreamException { + Double by = null; + + for (int i = 0; i < reader.getAttributeCount(); i++) { + if ("by".equals(reader.getAttributeName(i).toString())) { + by = Double.parseDouble(reader.getAttributeValue(i)); + } + } + + if (by == null) { + throw newDeserializeException("Assignment update without \"by\" attribute"); + } + + FieldValue key = null; + do { + reader.next(); + if (reader.getEventType() == XMLStreamReader.START_ELEMENT) { + if ("key".equals(reader.getName().toString())) { + if (f.getDataType() instanceof WeightedSetDataType) { + DataType nestedType = ((WeightedSetDataType)f.getDataType()).getNestedType(); + key = nestedType.createFieldValue(); + key.deserialize(this); + } else if (f.getDataType() instanceof MapDataType) { + key = ((MapDataType)f.getDataType()).getKeyType().createFieldValue(); + key.deserialize(this); + } else if (f.getDataType() instanceof ArrayDataType) { + key = new IntegerFieldValue(Integer.parseInt(reader.getElementText())); + } else { + throw newDeserializeException("Key tag only applicable for weighted sets and maps"); + } + skipToEnd("key"); + } else { + throw newDeserializeException("\"" + reader.getName() + "\" not appropriate within " + type + " element."); + } + } + } while (reader.getEventType() != XMLStreamReader.END_ELEMENT); + + if (key != null) { + if ("increment".equals(type)) { fu.addValueUpdate(ValueUpdate.createIncrement(key, by)); } + if ("decrement".equals(type)) { fu.addValueUpdate(ValueUpdate.createDecrement(key, by)); } + if ("multiply".equals(type)) { fu.addValueUpdate(ValueUpdate.createMultiply(key, by)); } + if ("divide".equals(type)) { fu.addValueUpdate(ValueUpdate.createDivide(key, by)); } + } else { + if ("increment".equals(type)) { fu.addValueUpdate(ValueUpdate.createIncrement(by)); } + if ("decrement".equals(type)) { fu.addValueUpdate(ValueUpdate.createDecrement(by)); } + if ("multiply".equals(type)) { fu.addValueUpdate(ValueUpdate.createMultiply(by)); } + if ("divide".equals(type)) { fu.addValueUpdate(ValueUpdate.createDivide(by)); } + } + + return fu; + } + + public void read(FieldUpdate update) { + } + + public void read(FieldPathUpdate update) { + String whereClause = null; + String fieldPath = null; + + for (int i = 0; i < reader.getAttributeCount(); i++) { + if (reader.getAttributeName(i).toString().equals("where")) { + whereClause = reader.getAttributeValue(i); + } else if (reader.getAttributeName(i).toString().equals("fieldpath")) { + fieldPath = reader.getAttributeValue(i); + } + } + + if (fieldPath != null) { + update.setFieldPath(fieldPath); + } else { + throw newDeserializeException("Field path is required for document updates."); + } + + if (whereClause != null) { + try { + update.setWhereClause(whereClause); + } catch (ParseException e) { + throw newException(e); + } + } + } + + public void read(AssignFieldPathUpdate update) { + try { + for (int i = 0; i < reader.getAttributeCount(); i++) { + if (reader.getAttributeName(i).toString().equals("removeifzero")) { + update.setRemoveIfZero(Boolean.parseBoolean(reader.getAttributeValue(i))); + } else if (reader.getAttributeName(i).toString().equals("createmissingpath")) { + update.setCreateMissingPath(Boolean.parseBoolean(reader.getAttributeValue(i))); + } + } + DataType dt = update.getFieldPath().getResultingDataType(); + + if (dt instanceof NumericDataType) { + update.setExpression(reader.getElementText()); + } else { + FieldValue fv = dt.createFieldValue(); + fv.deserialize(resolveField(update), this); + update.setNewValue(fv); + } + } catch (XMLStreamException e) { + throw newException(e); + } + } + + public void read(AddFieldPathUpdate update) { + DataType dt = update.getFieldPath().getResultingDataType(); + FieldValue fv = dt.createFieldValue(); + fv.deserialize(resolveField(update), this); + update.setNewValues((Array)fv); + } + + public void read(RemoveFieldPathUpdate update) { + } + + private static Field resolveField(FieldPathUpdate update) { + String orig = update.getOriginalFieldPath(); + if (orig == null) { + return null; + } + FieldPath path = update.getFieldPath(); + if (path == null) { + return null; + } + DataType type = path.getResultingDataType(); + if (type == null) { + return null; + } + return new Field(orig, type); + } +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java new file mode 100644 index 00000000000..eae7e1320a2 --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespaxmlparser; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/net/jpountz/lz4/package-info.java b/document/src/main/java/net/jpountz/lz4/package-info.java new file mode 100644 index 00000000000..45fc2e031ab --- /dev/null +++ b/document/src/main/java/net/jpountz/lz4/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage(version = @Version(major = 1, minor = 3, micro = 0)) +package net.jpountz.lz4; +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; |