blob: 2141505374c9de902719288be2475c56a93a79e9 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.sentencepiece;
/**
* Builds a result from a sentencepiece tokenization by being called for each segment in reverse
*
* @param <RESULTTYPE> the type of result this produces
* @author bratseth
*/
abstract class ResultBuilder<RESULTTYPE> {
private final RESULTTYPE result;
ResultBuilder(RESULTTYPE result) {
this.result = result;
}
/** Called for each segment, starting from the last and working backwards */
abstract void add(int start, int end, SentencePieceAlgorithm.SegmentEnd[] segmentEnds);
RESULTTYPE result() {return result;}
void build(String input, SentencePieceAlgorithm.SegmentEnd[] segmentEnds, boolean collapseUnknowns) {
if (collapseUnknowns) {
int segmentEnd = input.length();
int collapsedSegmentEnd = segmentEnd;
while (segmentEnd > 0) {
if (segmentEnds[segmentEnd].type != TokenType.unknown ) {
if (collapsedSegmentEnd != segmentEnd) { // We have deferred an unknown collapsed segment
add(segmentEnd, collapsedSegmentEnd, segmentEnds);
}
add(segmentEnds[segmentEnd].segmentStart, segmentEnd, segmentEnds);
collapsedSegmentEnd = segmentEnds[segmentEnd].segmentStart;
}
segmentEnd = segmentEnds[segmentEnd].segmentStart;
}
}
else {
int segmentEnd = input.length();
while (segmentEnd > 0) {
add(segmentEnds[segmentEnd].segmentStart, segmentEnd, segmentEnds);
segmentEnd = segmentEnds[segmentEnd].segmentStart;
}
}
}
}
|
