1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
|
// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.document.json;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.google.common.annotations.Beta;
import com.google.common.base.Preconditions;
import com.yahoo.document.Document;
import com.yahoo.document.DocumentId;
import com.yahoo.document.DocumentOperation;
import com.yahoo.document.DocumentPut;
import com.yahoo.document.DocumentRemove;
import com.yahoo.document.DocumentType;
import com.yahoo.document.DocumentTypeManager;
import com.yahoo.document.DocumentUpdate;
import com.yahoo.document.Field;
import com.yahoo.document.TestAndSetCondition;
import com.yahoo.document.json.document.DocumentParser;
import com.yahoo.document.json.readers.DocumentParseInfo;
import com.yahoo.document.update.FieldUpdate;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
import static com.yahoo.document.json.document.DocumentParser.parseDocumentsFields;
import static com.yahoo.document.json.readers.AddRemoveCreator.createAdds;
import static com.yahoo.document.json.readers.AddRemoveCreator.createRemoves;
import static com.yahoo.document.json.readers.CompositeReader.populateComposite;
import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd;
import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart;
import static com.yahoo.document.json.readers.MapReader.UPDATE_MATCH;
import static com.yahoo.document.json.readers.MapReader.createMapUpdate;
import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate;
/**
* Initialize Vespa documents/updates/removes from an InputStream containing a
* valid JSON representation of a feed.
*
* @author Steinar Knutsen
* @since 5.1.25
*/
@Beta
public class JsonReader {
// Only used for testing.
public Optional<DocumentParseInfo> parseDocument() {
return DocumentParser.parseDocument(parser);
}
private static final String UPDATE_REMOVE = "remove";
private static final String UPDATE_ADD = "add";
private final JsonParser parser;
private final DocumentTypeManager typeManager;
private ReaderState state = ReaderState.AT_START;
enum ReaderState {
AT_START, READING, END_OF_FEED
}
public JsonReader(DocumentTypeManager typeManager, InputStream input, JsonFactory parserFactory) {
this.typeManager = typeManager;
try {
parser = parserFactory.createParser(input);
} catch (IOException e) {
state = ReaderState.END_OF_FEED;
throw new RuntimeException(e);
}
}
/**
* Reads a single operation. The operation is not expected to be part of an array. It only reads FIELDS.
* @param operationType the type of operation (update or put)
* @param docIdString document ID.
* @return the document
*/
public DocumentOperation readSingleDocument(DocumentParser.SupportedOperation operationType, String docIdString) {
DocumentId docId = new DocumentId(docIdString);
DocumentParseInfo documentParseInfo = parseDocumentsFields(parser, docId);
documentParseInfo.operationType = operationType;
DocumentOperation operation = createDocumentOperation(documentParseInfo.fieldsBuffer, documentParseInfo);
operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition));
return operation;
}
public DocumentOperation next() {
switch (state) {
case AT_START:
JsonToken t = nextToken(parser);
expectArrayStart(t);
state = ReaderState.READING;
break;
case END_OF_FEED:
return null;
case READING:
break;
}
Optional<DocumentParseInfo> documentParseInfo = DocumentParser.parseDocument(parser);
if (! documentParseInfo.isPresent()) {
state = ReaderState.END_OF_FEED;
return null;
}
DocumentOperation operation = createDocumentOperation(documentParseInfo.get().fieldsBuffer, documentParseInfo.get());
operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.get().condition));
return operation;
}
private DocumentOperation createDocumentOperation(TokenBuffer buffer, DocumentParseInfo documentParseInfo) {
DocumentType documentType = getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager);
final DocumentOperation documentOperation;
try {
switch (documentParseInfo.operationType) {
case PUT:
documentOperation = new DocumentPut(new Document(documentType, documentParseInfo.documentId));
readPut(buffer, (DocumentPut) documentOperation);
verifyEndState(buffer);
break;
case REMOVE:
documentOperation = new DocumentRemove(documentParseInfo.documentId);
break;
case UPDATE:
documentOperation = new DocumentUpdate(documentType, documentParseInfo.documentId);
readUpdate(buffer, (DocumentUpdate) documentOperation);
verifyEndState(buffer);
break;
default:
throw new IllegalStateException("Implementation out of sync with itself. This is a bug.");
}
} catch (JsonReaderException e) {
throw JsonReaderException.addDocId(e, documentParseInfo.documentId);
}
if (documentParseInfo.create.isPresent()) {
if (!(documentOperation instanceof DocumentUpdate)) {
throw new RuntimeException("Could not set create flag on non update operation.");
}
DocumentUpdate update = (DocumentUpdate) documentOperation;
update.setCreateIfNonExistent(documentParseInfo.create.get());
}
return documentOperation;
}
// Exposed for unit testing...
void readUpdate(TokenBuffer buffer, DocumentUpdate next) {
if (buffer.size() == 0) {
buffer.bufferObject(nextToken(parser), parser);
}
populateUpdateFromBuffer(buffer, next);
}
// Exposed for unit testing...
void readPut(TokenBuffer buffer, DocumentPut put) {
if (buffer.size() == 0) {
buffer.bufferObject(nextToken(parser), parser);
}
try {
populateComposite(buffer, put.getDocument());
} catch (JsonReaderException e) {
throw JsonReaderException.addDocId(e, put.getId());
}
}
private void verifyEndState(TokenBuffer buffer) {
Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation");
expectObjectEnd(buffer.currentToken());
Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation");
Preconditions.checkState(buffer.size() == 0, "Dangling data at end of operation");
}
private static void populateUpdateFromBuffer(TokenBuffer buffer, DocumentUpdate update) {
expectObjectStart(buffer.currentToken());
int localNesting = buffer.nesting();
JsonToken t = buffer.next();
while (localNesting <= buffer.nesting()) {
expectObjectStart(t);
String fieldName = buffer.currentName();
Field field = update.getType().getField(fieldName);
addFieldUpdates(buffer, update, field);
t = buffer.next();
}
}
private static void addFieldUpdates(TokenBuffer buffer, DocumentUpdate update, Field field) {
int localNesting = buffer.nesting();
FieldUpdate fieldUpdate = FieldUpdate.create(field);
buffer.next();
while (localNesting <= buffer.nesting()) {
switch (buffer.currentName()) {
case UPDATE_REMOVE:
createRemoves(buffer, field, fieldUpdate);
break;
case UPDATE_ADD:
createAdds(buffer, field, fieldUpdate);
break;
case UPDATE_MATCH:
fieldUpdate.addValueUpdate(createMapUpdate(buffer, field));
break;
default:
String action = buffer.currentName();
fieldUpdate.addValueUpdate(readSingleUpdate(buffer, field.getDataType(), action));
}
buffer.next();
}
update.addFieldUpdate(fieldUpdate);
}
public DocumentType readDocumentType(DocumentId docId) {
return getDocumentTypeFromString(docId.getDocType(), typeManager);
}
private static DocumentType getDocumentTypeFromString(String docTypeString, DocumentTypeManager typeManager) {
final DocumentType docType = typeManager.getDocumentType(docTypeString);
if (docType == null) {
throw new IllegalArgumentException(String.format("Document type %s does not exist", docTypeString));
}
return docType;
}
public static JsonToken nextToken(JsonParser parser) {
try {
return parser.nextValue();
} catch (IOException e) {
// Jackson is not able to recover from structural parse errors
// TODO Do we really need to set state on exception?
// state = ReaderState.END_OF_FEED;
throw new RuntimeException(e);
}
}
}
|
