linguistics/src/test/java/com/yahoo/language/process/NormalizationTestCase.java


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.process;

import com.yahoo.language.simple.SimpleLinguistics;
import org.junit.Test;

import static org.junit.Assert.assertEquals;

/**
 * @author Mathias Mølster Lidal
 */
public class NormalizationTestCase {

    private final Normalizer normalizer = new SimpleLinguistics().getNormalizer();

    @Test
    public void testEmptyStringNormalization() {
        assertEquals("", normalizer.normalize(""));
    }

    @Test
    public void testDoubleWidthAscii() {
        assertNormalize("\uff41\uff42\uff43\uff44\uff45\uff46\uff47\uff48\uff49", "abcdefghi");
    }

    @Test
    public void testLigature() {
        assertNormalize("\uFB01nance", "finance");
    }

    private void assertNormalize(String input, String exp) {
        assertEquals(exp, normalizer.normalize(input));
    }

}