summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTor Egge <Tor.Egge@online.no>2022-01-03 11:09:58 +0100
committerTor Egge <Tor.Egge@online.no>2022-01-03 16:44:03 +0100
commit3cf59210f64ecd41baeb7556288b7883da54c397 (patch)
tree8dacf07c94e617e87cd00087ae849e437d7a82b3
parent96ede726dd237b16ea2b3a0f1b1c1c0a9cce835e (diff)
Revert "Revert "Don't split interlinear annotations.""
-rw-r--r--juniper/src/vespa/juniper/sumdesc.cpp122
1 files changed, 113 insertions, 9 deletions
diff --git a/juniper/src/vespa/juniper/sumdesc.cpp b/juniper/src/vespa/juniper/sumdesc.cpp
index 969616423e8..fcee1eb605f 100644
--- a/juniper/src/vespa/juniper/sumdesc.cpp
+++ b/juniper/src/vespa/juniper/sumdesc.cpp
@@ -29,6 +29,10 @@ char printable_char(char c)
return c;
}
+constexpr ucs4_t il_ann_anchor = 0xfff9;
+constexpr ucs4_t il_ann_separator = 0xfffa;
+constexpr ucs4_t il_ann_terminator = 0xfffb;
+
bool wordchar(const unsigned char* s)
{
unsigned char c = *s;
@@ -40,6 +44,28 @@ bool wordchar(const unsigned char* s)
}
}
+bool wordchar_or_il_ann_char(const unsigned char* s, ucs4_t annotation_char)
+{
+ unsigned char c = *s;
+ if (c & 0x80) {
+ ucs4_t u = Fast_UnicodeUtil::GetUTF8Char(s);
+ return Fast_UnicodeUtil::IsWordChar(u) ||
+ u == annotation_char;
+ } else {
+ return isalnum(c);
+ }
+}
+
+bool wordchar_or_il_ann_anchor(const unsigned char* s)
+{
+ return wordchar_or_il_ann_char(s, il_ann_anchor);
+}
+
+bool wordchar_or_il_ann_terminator(const unsigned char* s)
+{
+ return wordchar_or_il_ann_char(s, il_ann_terminator);
+}
+
bool nonwordchar(const unsigned char* s)
{
unsigned char c = *s;
@@ -51,6 +77,35 @@ bool nonwordchar(const unsigned char* s)
}
}
+bool
+il_ann_char(const unsigned char* s, ucs4_t annotation_char)
+{
+ unsigned char c = *s;
+ if (c & 0x80) {
+ ucs4_t u = Fast_UnicodeUtil::GetUTF8Char(s);
+ return u == annotation_char;
+ } else {
+ return false;
+ }
+}
+
+bool
+il_ann_anchor_char(const unsigned char* s)
+{
+ return il_ann_char(s, il_ann_anchor);
+}
+
+bool
+il_ann_separator_char(const unsigned char* s)
+{
+ return il_ann_char(s, il_ann_separator);
+}
+
+bool
+il_ann_terminator_char(const unsigned char* s)
+{
+ return il_ann_char(s, il_ann_terminator);
+}
/* Move backwards/forwards from ptr (no longer than to start) in an
* UTF8 text until the beginning of the word or (if space, until
@@ -75,12 +130,24 @@ int complete_word(unsigned char* start, ssize_t length,
}
// Figure out if a word needs completion or if we are just going
- // to eliminate whitespace
+ // to eliminate whitespace. Consider sequence from interlinear
+ // annotation anchor to interlinear annotation terminator to be a
+ // word.
if (!wordchar(ptr)) {
- whitespace_elim = true;
- // Change direction of scan
- increment = -increment;
- chartest = wordchar;
+ if (increment > 0 && il_ann_anchor_char(ptr)) {
+ chartest = il_ann_terminator_char;
+ } else if (increment < 0 && il_ann_terminator_char(ptr)) {
+ chartest = il_ann_anchor_char;
+ } else {
+ whitespace_elim = true;
+ // Change direction of scan
+ increment = -increment;
+ if (increment > 0) {
+ chartest = wordchar_or_il_ann_anchor;
+ } else {
+ chartest = wordchar_or_il_ann_terminator;
+ }
+ }
} else {
// Found a wordchar at pointer
// If moving forwards, we need to check the previous character
@@ -89,12 +156,16 @@ int complete_word(unsigned char* start, ssize_t length,
const unsigned char* pre_ptr = ptr;
int cur_move = Fast_UnicodeUtil::UTF8move(start, length,
pre_ptr, -1);
- if (!wordchar(pre_ptr)) // Points at start of new word
+ if (!wordchar(pre_ptr) && !il_ann_terminator_char(pre_ptr)) // Points at start of new word
{
whitespace_elim = true;
// Change direction of scan
increment = -increment;
- chartest = wordchar;
+ if (increment > 0) {
+ chartest = wordchar_or_il_ann_anchor;
+ } else {
+ chartest = wordchar_or_il_ann_terminator;
+ }
ptr = pre_ptr;
moved += cur_move;
} else {
@@ -129,6 +200,34 @@ int complete_word(unsigned char* start, ssize_t length,
break;
}
if (chartest(ptr)) {
+ if (chartest == nonwordchar) {
+ if (il_ann_separator_char(ptr)) {
+ if (increment > 0) {
+ chartest = il_ann_terminator_char;
+ } else {
+ chartest = il_ann_anchor_char;
+ }
+ moved += cur_move;
+ continue;
+ } else if (il_ann_terminator_char(ptr)) {
+ if (increment < 0) {
+ chartest = il_ann_anchor_char;
+ }
+ moved += cur_move;
+ continue;
+ } else if (il_ann_anchor_char(ptr)) {
+ if (increment > 0) {
+ chartest = il_ann_terminator_char;
+ }
+ moved += cur_move;
+ continue;
+ }
+ } else if ((chartest == il_ann_anchor_char) ||
+ (chartest == il_ann_terminator_char)) {
+ chartest = nonwordchar;
+ moved += cur_move;
+ continue;
+ }
LOG(spam, "complete_word: Breaking at char %c/0x%x (%d)", printable_char(*ptr),
*ptr, cur_move);
// count this character (it is the first blank/wordchar)
@@ -141,7 +240,9 @@ int complete_word(unsigned char* start, ssize_t length,
break; // Found first blank/word char..
}
moved += cur_move;
- if (moved >= MAX_SCAN_WORD) {
+ if (moved >= MAX_SCAN_WORD &&
+ (chartest != il_ann_anchor_char) &&
+ (chartest != il_ann_terminator_char)) {
LOG(spam, "Word length extended max word length %d, "
"breaking at char 0x%x", MAX_SCAN_WORD, *ptr);
break;
@@ -486,8 +587,11 @@ int SummaryDesc::complete_extended_token(unsigned char* start, ssize_t length,
// Only a single connector character that connects word
// characters should lead us to include more words in the
// normal sense:
- if (!wordchar(preptr))
+ if (!wordchar(preptr) &&
+ !(increment > 0 && il_ann_anchor_char(preptr)) &&
+ !(increment < 0 && il_ann_terminator_char(preptr))) {
return moved;
+ }
// If a block of chinese data does not contain any spaces we have to return
// here in order to avoid searching all the way to the start/end.