diff options
author | Jon Marius Venstad <venstad@gmail.com> | 2021-12-17 15:59:23 +0100 |
---|---|---|
committer | Jon Marius Venstad <venstad@gmail.com> | 2021-12-17 15:59:23 +0100 |
commit | a8752c293953ad41a98d92475a05619fc3c09710 (patch) | |
tree | 1601aad7141c9dc2f40d687188f507499b453230 | |
parent | b57d7fab9aff90f432bf7f1a9663c608d7d7f880 (diff) |
Replace UrlcharSequenceNormalizer with one with an improved regex
5 files changed, 80 insertions, 6 deletions
diff --git a/linguistics-components/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java b/linguistics-components/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java new file mode 100644 index 00000000000..aa4387bcc45 --- /dev/null +++ b/linguistics-components/src/main/java/com/yahoo/language/opennlp/LanguageDetectorFactory.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.langdetect.DefaultLanguageDetectorContextGenerator; +import opennlp.tools.langdetect.LanguageDetectorContextGenerator; +import opennlp.tools.util.normalizer.EmojiCharSequenceNormalizer; +import opennlp.tools.util.normalizer.NumberCharSequenceNormalizer; +import opennlp.tools.util.normalizer.ShrinkCharSequenceNormalizer; +import opennlp.tools.util.normalizer.TwitterCharSequenceNormalizer; + +/** + * Overrides the UrlCharSequenceNormalizer, which has a bad regex, until fixed: https://issues.apache.org/jira/browse/OPENNLP-1350 + * + * @author jonmv + */ +@SuppressWarnings("unused") // Loaded by black magic. +public class LanguageDetectorFactory extends opennlp.tools.langdetect.LanguageDetectorFactory { + + @Override + public LanguageDetectorContextGenerator getContextGenerator() { + return new DefaultLanguageDetectorContextGenerator(1, 3, + EmojiCharSequenceNormalizer.getInstance(), + UrlCharSequenceNormalizer.getInstance(), + TwitterCharSequenceNormalizer.getInstance(), + NumberCharSequenceNormalizer.getInstance(), + ShrinkCharSequenceNormalizer.getInstance()); + } + +} diff --git a/linguistics-components/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java b/linguistics-components/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java new file mode 100644 index 00000000000..883319e2f8b --- /dev/null +++ b/linguistics-components/src/main/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizer.java @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import opennlp.tools.util.normalizer.CharSequenceNormalizer; + +import java.util.regex.Pattern; + +/** + * Modifies {@link opennlp.tools.util.normalizer.UrlCharSequenceNormalizer} to avoid the bad email regex. + * + * @author jonmv + */ +public class UrlCharSequenceNormalizer implements CharSequenceNormalizer { + + private static final Pattern URL_REGEX = + Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]+"); + private static final Pattern MAIL_REGEX = + Pattern.compile("(?<![-+_.0-9A-Za-z])[-+_.0-9A-Za-z]+@[-0-9A-Za-z]+[-.0-9A-Za-z]+"); + + private static final UrlCharSequenceNormalizer INSTANCE = new UrlCharSequenceNormalizer(); + + public static UrlCharSequenceNormalizer getInstance() { + return INSTANCE; + } + + public CharSequence normalize(CharSequence text) { + String modified = URL_REGEX.matcher(text).replaceAll(" "); + return MAIL_REGEX.matcher(modified).replaceAll(" "); + } + +} diff --git a/linguistics-components/src/main/resources/models/langdetect-183.bin b/linguistics-components/src/main/resources/models/langdetect-183.bin Binary files differindex 0b4ea89690e..c3cde217050 100644 --- a/linguistics-components/src/main/resources/models/langdetect-183.bin +++ b/linguistics-components/src/main/resources/models/langdetect-183.bin diff --git a/linguistics-components/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java b/linguistics-components/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java new file mode 100644 index 00000000000..a8c637bc6ec --- /dev/null +++ b/linguistics-components/src/test/java/com/yahoo/language/opennlp/UrlCharSequenceNormalizerTest.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.opennlp; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class UrlCharSequenceNormalizerTest { + + @Test + public void testNormalization() { + String text = "xxx+yyy_.dude@mail.com foo bar@baz_bax https://host.tld/path?query=boo a@b §boo@boo"; + assertEquals(" foo _bax a@b § ", + UrlCharSequenceNormalizer.getInstance().normalize(text)); + } + +} diff --git a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def b/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def deleted file mode 100644 index 361a8a5f50c..00000000000 --- a/linguistics/src/main/resources/configdefinitions/language.opennlp.opennlp-linguistics.def +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=language.opennlp - -# Enable Optimaize language detector -detector.enableOptimaize bool default=true - |