blob: 7f48537f1794c738a38ed1c4fad5bb57d5f7953d (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include <vespa/vespalib/testkit/test_kit.h>
#include <vespa/vespalib/testkit/testapp.h>
#include <vespa/vespalib/text/utf8.h>
#include <fcntl.h>
#include <unistd.h>
#include <vespa/log/log.h>
LOG_SETUP("utf8_test");
#if 0
#include <vespa/fastlib/text/unicodeutil.h>
#endif
using namespace vespalib;
TEST_SETUP(Test);
int
Test::Main()
{
TEST_INIT("utf8_test");
for (uint32_t h = 0; h < 0x1100; h++) {
vespalib::string data;
if (h >= 0xD8 && h < 0xE0) continue;
Utf8Writer w(data);
for (uint32_t i = 0; i < 256; i++) {
unsigned int codepoint = (h << 8) | i;
w.putChar(codepoint);
}
fprintf(stderr, "encoded 256 codepoints [U+%04X,U+%04X] in %zu bytes\n",
(h << 8), (h << 8) | 0xFF, data.size());
Utf8Reader r(data);
for (uint32_t i = 0; i < 256; i++) {
EXPECT_TRUE(r.hasMore());
unsigned int codepoint = (h << 8) | i;
unsigned int got = r.getChar(12345678);
EXPECT_EQUAL(codepoint, got);
}
EXPECT_TRUE(! r.hasMore());
#if 0
char *p = data.begin();
char *e = data.end();
for (uint32_t i = 0; i < 256; i++) {
unsigned int codepoint = (h << 8) | i;
unsigned int got = Fast_UnicodeUtil::GetUTF8Char(p);
EXPECT_EQUAL(codepoint, got);
}
EXPECT_EQUAL(p, e);
#endif
}
{
// read data produced from Java program
int fd = ::open(TEST_PATH("regular-utf8.dat").c_str(), O_RDONLY);
ASSERT_TRUE(fd > 0);
vespalib::string data;
data.clear();
data.reserve(5510);
ASSERT_TRUE(::read(fd, data.begin(), 5510) == 5509);
data.append_from_reserved(5509);
Utf8Reader r(data);
uint32_t i = 32;
uint32_t j = 3;
while (i < 0x110000) {
if (i < 0xD800 || i >= 0xE000) {
ASSERT_TRUE(r.hasMore());
uint32_t got = r.getChar(12345678);
EXPECT_EQUAL(i, got);
}
i += j;
j++;
}
EXPECT_TRUE(! r.hasMore());
}
TEST_DONE();
}
|
