dungba88 commented on code in PR #12915:
URL: https://github.com/apache/lucene/pull/12915#discussion_r1423383461
##########
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaUppercaseFilter.java:
##########
@@ -0,0 +1,83 @@
+package org.apache.lucene.analysis.ja;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/**
+ * A {@link TokenFilter} that normalizes small letters (捨て仮名) in katakana into
normal letters. For
+ * instance, "ストップウォッチ" will be translated to "ストツプウオツチ".
+ *
+ * <p>This filter is useful if you want to search against old style Japanese
text such as patents,
+ * legal, contract policies, etc.
+ */
+public final class JapaneseKatakanaUppercaseFilter extends TokenFilter {
+ private static final Map<Character, Character> s2l;
+
+ static {
+ // supported characters are:
+ // ァ ィ ゥ ェ ォ ヵ ㇰ ヶ ㇱ ㇲ ッ ㇳ ㇴ ㇵ ㇶ ㇷ ㇷ゚ ㇸ ㇹ ㇺ ャ ュ ョ ㇻ ㇼ ㇽ ㇾ ㇿ ヮ
+ s2l =
+ Map.ofEntries(
+ Map.entry('ァ', 'ア'),
+ Map.entry('ィ', 'イ'),
+ Map.entry('ゥ', 'ウ'),
+ Map.entry('ェ', 'エ'),
+ Map.entry('ォ', 'オ'),
+ Map.entry('ヵ', 'カ'),
+ Map.entry('ㇰ', 'ク'),
+ Map.entry('ヶ', 'ケ'),
+ Map.entry('ㇱ', 'シ'),
+ Map.entry('ㇲ', 'ス'),
+ Map.entry('ッ', 'ツ'),
+ Map.entry('ㇳ', 'ト'),
+ Map.entry('ㇴ', 'ヌ'),
+ Map.entry('ㇵ', 'ハ'),
+ Map.entry('ㇶ', 'ヒ'),
+ Map.entry('ㇷ', 'フ'),
+ Map.entry('ㇸ', 'ヘ'),
+ Map.entry('ㇹ', 'ホ'),
+ Map.entry('ㇺ', 'ム'),
+ Map.entry('ャ', 'ヤ'),
+ Map.entry('ュ', 'ユ'),
+ Map.entry('ョ', 'ヨ'),
+ Map.entry('ㇻ', 'ラ'),
+ Map.entry('ㇼ', 'リ'),
+ Map.entry('ㇽ', 'ル'),
+ Map.entry('ㇾ', 'レ'),
+ Map.entry('ㇿ', 'ロ'),
+ Map.entry('ヮ', 'ワ'));
+ }
+
+ private final CharTermAttribute termAttr =
addAttribute(CharTermAttribute.class);
+
+ public JapaneseKatakanaUppercaseFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ String term = termAttr.toString();
+ // Small letter "ㇷ゚" is not single character, so it should be converted
to "プ" as String
+ term = term.replace("ㇷ゚", "プ");
+ char[] src = term.toCharArray();
Review Comment:
The buffer return the internal byte[] of the CharTermAttribute, which might
has more bytes than the actual term length. You need to use term.length() as
well.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
