1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.document.json;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.yahoo.document.DocumentId;
import com.yahoo.document.DocumentOperation;
import com.yahoo.document.DocumentPut;
import com.yahoo.document.DocumentType;
import com.yahoo.document.DocumentTypeManager;
import com.yahoo.document.DocumentUpdate;
import com.yahoo.document.TestAndSetCondition;
import com.yahoo.document.json.document.DocumentParser;
import com.yahoo.document.json.readers.DocumentParseInfo;
import com.yahoo.document.json.readers.VespaJsonDocumentReader;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
import static com.yahoo.document.json.JsonReader.ReaderState.END_OF_FEED;
import static com.yahoo.document.json.document.DocumentParser.CONDITION;
import static com.yahoo.document.json.document.DocumentParser.CREATE_IF_NON_EXISTENT;
import static com.yahoo.document.json.document.DocumentParser.FIELDS;
import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart;
/**
* Initialize Vespa documents/updates/removes from an InputStream containing a
* valid JSON representation of a feed.
*
* @author Steinar Knutsen
* @author Haakon Dybdahl
*/
public class JsonReader {
private final JsonParser parser;
private final DocumentTypeManager typeManager;
private ReaderState state = ReaderState.AT_START;
enum ReaderState {
AT_START, READING, END_OF_FEED
}
public JsonReader(DocumentTypeManager typeManager, InputStream input, JsonFactory parserFactory) {
this.typeManager = typeManager;
try {
parser = parserFactory.createParser(input);
} catch (IOException e) {
state = END_OF_FEED;
throw new IllegalArgumentException(e);
}
}
public Optional<DocumentParseInfo> parseDocument() throws IOException {
DocumentParser documentParser = new DocumentParser(parser);
return documentParser.parse(Optional.empty());
}
/**
* Reads a single operation. The operation is not expected to be part of an array.
*
* @param operationType the type of operation (update or put)
* @param docIdString document ID
* @return the parsed document operation
*/
ParsedDocumentOperation readSingleDocument(DocumentOperationType operationType, String docIdString) {
DocumentId docId = new DocumentId(docIdString);
DocumentParseInfo documentParseInfo;
try {
DocumentParser documentParser = new DocumentParser(parser);
documentParseInfo = documentParser.parse(Optional.of(docId)).get();
} catch (IOException e) {
state = END_OF_FEED;
throw new IllegalArgumentException(e);
}
documentParseInfo.operationType = operationType;
VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields());
ParsedDocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation(
getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager), documentParseInfo);
operation.operation().setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition));
return operation;
}
/**
* Reads a JSON which is expected to contain a single document operation,
* and where other parameters, like the document ID and operation type, are supplied by other means.
*
* @param operationType the type of operation (update or put)
* @param docIdString document ID
* @return the parsed document operation
*/
public ParsedDocumentOperation readSingleDocumentStreaming(DocumentOperationType operationType, String docIdString) {
try {
DocumentId docId = new DocumentId(docIdString);
DocumentParseInfo documentParseInfo = new DocumentParseInfo();
documentParseInfo.documentId = docId;
documentParseInfo.operationType = operationType;
if (JsonToken.START_OBJECT != parser.nextValue())
throw new IllegalArgumentException("expected start of root object, got " + parser.currentToken());
Boolean create = null;
String condition = null;
ParsedDocumentOperation operation = null;
while (JsonToken.END_OBJECT != parser.nextValue()) {
switch (parser.currentName()) {
case FIELDS -> {
documentParseInfo.fieldsBuffer = new LazyTokenBuffer(parser);
VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields());
operation = vespaJsonDocumentReader.createDocumentOperation(
getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager), documentParseInfo);
if ( ! documentParseInfo.fieldsBuffer.isEmpty())
throw new IllegalArgumentException("expected all content to be consumed by document parsing, but " +
documentParseInfo.fieldsBuffer.nesting() + " levels remain");
}
case CONDITION -> {
if ( ! JsonToken.VALUE_STRING.equals(parser.currentToken()) && ! JsonToken.VALUE_NULL.equals(parser.currentToken()))
throw new IllegalArgumentException("expected string value for condition, got " + parser.currentToken());
condition = parser.getValueAsString();
}
case CREATE_IF_NON_EXISTENT -> {
create = parser.getBooleanValue(); // Throws if not boolean.
}
default -> {
// We ignore stray fields, but need to ensure structural balance in doing do.
if (parser.currentToken().isStructStart()) parser.skipChildren();
}
}
}
if (null != parser.nextToken())
throw new IllegalArgumentException("expected end of input, got " + parser.currentToken());
if (null == operation)
throw new IllegalArgumentException("document is missing the required \"fields\" field");
if (null != create) {
switch (operationType) {
case PUT -> ((DocumentPut) operation.operation()).setCreateIfNonExistent(create);
case UPDATE -> ((DocumentUpdate) operation.operation()).setCreateIfNonExistent(create);
case REMOVE -> throw new IllegalArgumentException(CREATE_IF_NON_EXISTENT + " is not supported for remove operations");
}
}
operation.operation().setCondition(TestAndSetCondition.fromConditionString(Optional.ofNullable(condition)));
return operation;
}
catch (IOException e) {
throw new IllegalArgumentException("failed parsing document", e);
}
}
/** Returns the next document operation, or null if we have reached the end */
public DocumentOperation next() {
switch (state) {
case AT_START:
JsonToken t = nextToken(parser);
expectArrayStart(t);
state = ReaderState.READING;
break;
case END_OF_FEED:
return null;
case READING:
break;
}
Optional<DocumentParseInfo> documentParseInfo;
try {
documentParseInfo = parseDocument();
} catch (IOException r) {
// Jackson is not able to recover from structural parse errors
state = END_OF_FEED;
throw new IllegalArgumentException(r);
}
if (documentParseInfo.isEmpty()) {
state = END_OF_FEED;
return null;
}
VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields());
DocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation(
getDocumentTypeFromString(documentParseInfo.get().documentId.getDocType(), typeManager),
documentParseInfo.get()).operation();
operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.get().condition));
return operation;
}
public DocumentType readDocumentType(DocumentId docId) {
return getDocumentTypeFromString(docId.getDocType(), typeManager);
}
private static DocumentType getDocumentTypeFromString(String docTypeString, DocumentTypeManager typeManager) {
final DocumentType docType = typeManager.getDocumentType(docTypeString);
if (docType == null)
throw new IllegalArgumentException(String.format("Document type %s does not exist", docTypeString));
return docType;
}
public JsonToken nextToken(JsonParser parser) {
try {
return parser.nextValue();
} catch (IOException e) {
// Jackson is not able to recover from structural parse errors
state = END_OF_FEED;
throw new IllegalArgumentException(e);
}
}
}
|
