1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.document.serialization;
import com.yahoo.document.ArrayDataType;
import com.yahoo.document.DataType;
import com.yahoo.document.DocumentTypeManager;
import com.yahoo.document.DocumentTypeManagerConfigurer;
import com.yahoo.document.Field;
import com.yahoo.document.StructDataType;
import com.yahoo.document.annotation.*;
import com.yahoo.document.datatypes.Array;
import com.yahoo.document.datatypes.DoubleFieldValue;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.StringFieldValue;
import com.yahoo.document.datatypes.Struct;
import com.yahoo.io.GrowableByteBuffer;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.util.ArrayList;
import static org.junit.Assert.assertEquals;
/**
* Tests that serialization of annotations from Java generates the
* output expected by the C++ deserialization system.
*
* If the format changes and this test starts failing, you should
* generate new golden files to check against. This will cause the C++
* test to fail, so you will need to update the
* AnnotationDeserialization component to handle the format changes.
*/
public class SerializeAnnotationsTestCase {
private static final String PATH = "src/tests/serialization/";
DocumentTypeManager docMan = new DocumentTypeManager();
@Before
public void setUp() {
var sub = DocumentTypeManagerConfigurer.configure
(docMan, "file:src/tests/serialization/annotation.serialize.test.cfg");
sub.close();
}
@Test
public void testSerializeSimpleTree() throws IOException {
SpanList root = new SpanList();
root.add(new Span(0, 19))
.add(new Span(19, 5))
.add(new Span(24, 21))
.add(new Span(45, 23))
.add(new Span(68, 14));
SpanTree tree = new SpanTree("html", root);
StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj " +
"lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj l jlkj lkj lkj " +
"lkjoijoij oij oij oij oij oij oijoijoij oij oij oij oij oij " +
"oijoijoijoijoijoijoijoijoijoijoijoijoij oij oij oij oij " +
"oijaosdifjoai fdoais jdoasi jai os oafoai ai dfojsfoa dfoi dsf" +
"aosifjasofija sodfij oasdifj aosdiosifjsi ooai oais osi");
value.setSpanTree(tree);
/*
Important note! The iteration order of annotations in SpanTree.iterator() is non-deterministic, meaning
that the order which annotations are serialized will differ between test runs. Thus, we cannot assert
that a serialized buffer is equal to a buffer written earlier. We can, however, assert that the size stays
the same, and the deserialized values from the buffers should be equal.
*/
//important! call readFile() before writeFile()!
ByteBuffer serializedFromFile = readFile("test_data_serialized_simple");
ByteBuffer serialized = writeFile(value, "test_data_serialized_simple");
assertEquals(serialized.limit(), serializedFromFile.limit());
StringFieldValue valueFromFile = new StringFieldValue();
DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, new GrowableByteBuffer(serializedFromFile));
deserializer.read(null, valueFromFile);
assertEquals(value, valueFromFile);
}
@Test
public void testSerializeAdvancedTree() throws IOException {
SpanList root = new SpanList();
SpanTree tree = new SpanTree("html", root);
AnnotationTypeRegistry registry = docMan.getAnnotationTypeRegistry();
AnnotationType textType = registry.getType("text");
AnnotationType beginTag = registry.getType("begintag");
AnnotationType endTag = registry.getType("endtag");
AnnotationType bodyType = registry.getType("body");
AnnotationType paragraphType = registry.getType("paragraph");
AnnotationType cityType = registry.getType("city");
var cityDataType = (StructDataType) cityType.getDataType();
var positionType = (StructDataType) cityDataType.getField("position").getDataType();
var refArrT = (ArrayDataType) cityDataType.getField("references").getDataType();
var annRefType = (AnnotationReferenceDataType) refArrT.getNestedType();
Struct position = new Struct(positionType);
position.setFieldValue("latitude", new DoubleFieldValue(37.774929));
position.setFieldValue("longitude", new DoubleFieldValue(-122.419415));
Annotation sanAnnotation = new Annotation(textType);
Annotation franciscoAnnotation = new Annotation(textType);
Struct positionWithRef = cityDataType.createFieldValue();
positionWithRef.setFieldValue("position", position);
Field referencesField = cityDataType.getField("references");
Array<FieldValue> refList =
new Array<FieldValue>(referencesField.getDataType());
refList.add(new AnnotationReference(annRefType, sanAnnotation));
refList.add(new AnnotationReference(annRefType, franciscoAnnotation));
positionWithRef.setFieldValue(referencesField, refList);
Annotation city = new Annotation(cityType, positionWithRef);
AlternateSpanList paragraph = new AlternateSpanList();
paragraph.addChildren(new ArrayList<SpanNode>(), 0);
paragraph.setProbability(0, 0.9);
paragraph.setProbability(1, 0.1);
{
Span span1 = new Span(6, 3);
Span span2 = new Span(9, 10);
Span span3 = new Span(19, 4);
Span span4 = new Span(23, 4);
paragraph.add(0, span1)
.add(0, span2)
.add(0, span3)
.add(0, span4);
Span alt_span1 = new Span(6, 13);
Span alt_span2 = new Span(19, 8);
paragraph.add(1, alt_span1)
.add(1, alt_span2);
tree.annotate(span1, beginTag)
.annotate(span2, textType)
.annotate(span3, sanAnnotation)
.annotate(span4, endTag)
.annotate(alt_span1, textType)
.annotate(alt_span2, bodyType)
.annotate(paragraph, paragraphType);
}
{
Span span1 = new Span(0, 6);
Span span2 = new Span(27, 9);
Span span3 = new Span(36, 8);
root.add(span1)
.add(paragraph)
.add(span2)
.add(span3);
tree.annotate(span1, beginTag)
.annotate(span2, franciscoAnnotation)
.annotate(span3, endTag)
.annotate(root, bodyType)
.annotate(city);
}
StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj " +
"lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj l jlkj lkj lkj " +
"lkjoijoij oij oij oij oij oij oijoijoij oij oij oij oij oij " +
"oijoijoijoijoijoijoijoijoijoijoijoijoij oij oij oij oij " +
"oijaosdifjoai fdoais jdoasi jai os oafoai ai dfojsfoa dfoi dsf" +
"aosifjasofija sodfij oasdifj aosdiosifjsi ooai oais osi");
value.setSpanTree(tree);
//important! call readFile() before writeFile()!
ByteBuffer serializedFromFile = readFile("test_data_serialized_advanced");
ByteBuffer serialized = writeFile(value, "test_data_serialized_advanced");
assertEquals(serialized.limit(), serializedFromFile.limit());
StringFieldValue valueFromFile = new StringFieldValue();
DocumentDeserializer deserializer = DocumentDeserializerFactory.createHead(docMan, new GrowableByteBuffer(serializedFromFile));
deserializer.read(null, valueFromFile);
assertEquals(value, valueFromFile);
}
private static ByteBuffer writeFile(StringFieldValue value, String fileName) throws IOException {
fileName = PATH + fileName;
//serialize our tree to buffer
DocumentSerializer serializer = DocumentSerializerFactory.create6();
serializer.write(null, value);
ByteBuffer serializedBuf = serializer.getBuf().getByteBuffer();
serializedBuf.flip();
//write our tree to disk
File file = new File(fileName);
FileChannel wChannel = new FileOutputStream(file, false).getChannel();
wChannel.write(serializedBuf);
wChannel.close();
serializedBuf.position(0);
return serializedBuf;
}
private static ByteBuffer readFile(String fileName) throws IOException {
fileName = PATH + fileName;
//read tree from disk
ReadableByteChannel channel = new FileInputStream(fileName).getChannel();
ByteBuffer readBuf = ByteBuffer.allocate(4096);
channel.read(readBuf);
readBuf.flip();
channel.close();
return readBuf;
}
}
|
