1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vdslib;
import com.yahoo.compress.CompressionType;
import com.yahoo.document.BucketIdFactory;
import com.yahoo.document.DataType;
import com.yahoo.document.Document;
import com.yahoo.document.DocumentPut;
import com.yahoo.document.DocumentRemove;
import com.yahoo.document.DocumentUpdate;
import com.yahoo.document.serialization.DocumentSerializer;
import com.yahoo.document.serialization.DocumentSerializerFactory;
import com.yahoo.vespa.objects.Serializer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* A list of document operations.
*
* @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
*/
public class DynamicDocumentList extends DocumentList {
private List<Entry> entries;
DynamicDocumentList(List<Entry> entries) {
//the entries themselves are of course still modifiable, this is just an internal safeguard:
this.entries = Collections.unmodifiableList(entries);
}
DynamicDocumentList(Entry entry) {
List<Entry> list = new ArrayList<>(1);
list.add(entry);
BucketIdFactory factory = new BucketIdFactory();
//the entry itself is of course still modifiable, this is just an internal safeguard:
this.entries = Collections.unmodifiableList(list);
}
@Override
public Entry get(int index) throws ArrayIndexOutOfBoundsException {
return entries.get(index);
}
@Override
public int size() {
return entries.size();
}
@Override
public int getApproxByteSize() {
int size = 4;
for (Entry entry : entries) {
if (entry.getDocumentOperation() instanceof DocumentPut) {
Document doc = ((DocumentPut)entry.getDocumentOperation()).getDocument();
size += MetaEntry.SIZE + doc.getSerializedSize();
} else if (entry.getDocumentOperation() instanceof DocumentUpdate) {
//TODO: Implement getSerializedSize() for DocumentUpdate!!!
size += MetaEntry.SIZE + 1024;
} else if (entry.getDocumentOperation() instanceof DocumentRemove) {
//TODO: Implement getSerializedSize() for DocumentRemove!!!
size += MetaEntry.SIZE + 64;
}
}
return size;
}
@Override
public void serialize(Serializer buf) {
if (buf instanceof DocumentSerializer) {
serializeInternal((DocumentSerializer) buf);
} else {
DocumentSerializer serializer = DocumentSerializerFactory.create42();
serializeInternal(serializer);
serializer.getBuf().getByteBuffer().flip();
buf.put(null, serializer.getBuf().getByteBuffer());
}
}
@SuppressWarnings("deprecation")
private void serializeInternal(DocumentSerializer buf) {
ByteOrder originalOrder = buf.getBuf().order();
buf.getBuf().order(ByteOrder.LITTLE_ENDIAN);
//save the position before the size
int posAtBeginning = buf.getBuf().position();
//write the number of entries
buf.putInt(null, entries.size());
//create a list of metaentries, one for each entry
List<MetaEntry> metaEntries = new ArrayList<MetaEntry>(entries.size());
//jump past the meta block, we will serialize this afterwards when we know sizes and positions
byte[] bogusEntry = new byte[entries.size() * MetaEntry.SIZE];
buf.put(null, bogusEntry);
for (Entry entry : entries) {
MetaEntry metaEntry = new MetaEntry();
metaEntries.add(metaEntry);
// is this a remove? in that case, set this flag
if (entry.isRemoveEntry()) metaEntry.flags |= MetaEntry.REMOVE_ENTRY;
// is the body stripped? in that case, set this flag
if (entry.isBodyStripped()) metaEntry.flags |= MetaEntry.BODY_STRIPPED;
// is this an update? in that case, set this flag
if (entry.getDocumentOperation() instanceof DocumentUpdate) metaEntry.flags |= MetaEntry.UPDATE_ENTRY;
// is this a document? in that case, try to set the timestamp
if (entry.getDocumentOperation() instanceof DocumentPut) {
Document doc = ((DocumentPut)entry.getDocumentOperation()).getDocument();
Long lastModified = doc.getLastModified();
if (lastModified != null) {
metaEntry.timestamp = lastModified;
}
if (doc.getDataType().contentStruct().getCompressionConfig() != null
&& doc.getDataType().contentStruct().getCompressionConfig().type != CompressionType.NONE) {
metaEntry.flags |= MetaEntry.COMPRESSED;
}
if (doc.getDataType().getBodyType().getCompressionConfig() != null
&& doc.getDataType().getBodyType().getCompressionConfig().type != CompressionType.NONE) {
metaEntry.flags |= MetaEntry.COMPRESSED;
}
}
metaEntry.headerPos = buf.getBuf().position() - posAtBeginning;
buf.getBuf().order(ByteOrder.BIG_ENDIAN);
if (entry.getDocumentOperation() instanceof DocumentPut) {
Document doc = ((DocumentPut)entry.getDocumentOperation()).getDocument();
//serialize document and save length:
doc.serializeHeader(buf);
} else if (entry.getDocumentOperation() instanceof DocumentUpdate) {
DocumentUpdate docUp = (DocumentUpdate) entry.getDocumentOperation();
docUp.serialize(buf);
} else if (entry.getDocumentOperation() instanceof DocumentRemove) {
new Document(DataType.DOCUMENT, entry.getDocumentOperation().getId()).serialize(buf);
} else {
throw new IllegalArgumentException("Can not handle class " + entry.getDocumentOperation().getClass().getName());
}
metaEntry.headerLen = buf.getBuf().position() - metaEntry.headerPos - posAtBeginning;
if (entry.getDocumentOperation() instanceof DocumentPut) {
metaEntry.bodyPos = buf.getBuf().position() - posAtBeginning;
Document doc = ((DocumentPut)entry.getDocumentOperation()).getDocument();
doc.serializeBody(buf);
metaEntry.bodyLen = buf.getBuf().position() - metaEntry.bodyPos - posAtBeginning;
} else {
metaEntry.bodyPos = 0;
metaEntry.bodyLen = 0;
}
buf.getBuf().order(ByteOrder.LITTLE_ENDIAN);
}
//save position after payload:
int posAfterEntries = buf.getBuf().position();
//go to beginning (after length) to serialize metaentries:
buf.getBuf().position(posAtBeginning + 4);
//serialize metaentries
for (MetaEntry metaEntry : metaEntries) {
metaEntry.serialize(buf.getBuf());
}
//set position to after payload:
buf.getBuf().position(posAfterEntries);
buf.getBuf().order(originalOrder);
}
}
|
