mikemccand commented on code in PR #12915:
URL: https://github.com/apache/lucene/pull/12915#discussion_r1422804214
##########
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseHiraganaUppercaseFilter.java:
##########
@@ -0,0 +1,65 @@
+package org.apache.lucene.analysis.ja;
Review Comment:
Could you please add the standard Apache copyright header, if that's OK with
you? Thanks! I think this will also make the GitHub actions checks
(`./gradlew check`) happy.
##########
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseHiraganaUppercaseFilter.java:
##########
@@ -0,0 +1,65 @@
+package org.apache.lucene.analysis.ja;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/**
+ * A {@link TokenFilter} that normalizes small letters (捨て仮名) in hiragana into
normal letters. For
+ * instance, "ちょっとまって" will be translated to "ちよつとまつて".
+ *
+ * <p>This filter is useful if you want to search against old style Japanese
text such as patents,
+ * legal, contract policies, etc.
+ */
+public final class JapaneseHiraganaUppercaseFilter extends TokenFilter {
+ private static final Map<Character, Character> s2l;
+
+ static {
+ // supported characters are:
+ // ぁ ぃ ぅ ぇ ぉ っ ゃ ゅ ょ ゎ ゕ ゖ
+ s2l =
+ Map.ofEntries(
+ Map.entry('ぁ', 'あ'),
+ Map.entry('ぃ', 'い'),
+ Map.entry('ぅ', 'う'),
+ Map.entry('ぇ', 'え'),
+ Map.entry('ぉ', 'お'),
+ Map.entry('っ', 'つ'),
+ Map.entry('ゃ', 'や'),
+ Map.entry('ゅ', 'ゆ'),
+ Map.entry('ょ', 'よ'),
+ Map.entry('ゎ', 'わ'),
+ Map.entry('ゕ', 'か'),
+ Map.entry('ゖ', 'け'));
+ }
+
+ private final CharTermAttribute termAttr =
addAttribute(CharTermAttribute.class);
+
+ public JapaneseHiraganaUppercaseFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ String term = termAttr.toString();
+ char[] src = term.toCharArray();
+ char[] result = new char[src.length];
+ for (int i = 0; i < src.length; i++) {
+ Character c = s2l.get(src[i]);
+ if (c != null) {
+ result[i] = c;
+ } else {
+ result[i] = src[i];
+ }
+ }
+ String resultTerm = String.copyValueOf(result);
+ termAttr.setEmpty().append(resultTerm);
Review Comment:
You can avoid making `String` here by appending the `char[] result` instead.
##########
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaUppercaseFilter.java:
##########
@@ -0,0 +1,83 @@
+package org.apache.lucene.analysis.ja;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/**
+ * A {@link TokenFilter} that normalizes small letters (捨て仮名) in katakana into
normal letters. For
+ * instance, "ストップウォッチ" will be translated to "ストツプウオツチ".
+ *
+ * <p>This filter is useful if you want to search against old style Japanese
text such as patents,
+ * legal, contract policies, etc.
+ */
+public final class JapaneseKatakanaUppercaseFilter extends TokenFilter {
+ private static final Map<Character, Character> s2l;
+
+ static {
+ // supported characters are:
+ // ァ ィ ゥ ェ ォ ヵ ㇰ ヶ ㇱ ㇲ ッ ㇳ ㇴ ㇵ ㇶ ㇷ ㇷ゚ ㇸ ㇹ ㇺ ャ ュ ョ ㇻ ㇼ ㇽ ㇾ ㇿ ヮ
+ s2l =
+ Map.ofEntries(
+ Map.entry('ァ', 'ア'),
+ Map.entry('ィ', 'イ'),
+ Map.entry('ゥ', 'ウ'),
+ Map.entry('ェ', 'エ'),
+ Map.entry('ォ', 'オ'),
+ Map.entry('ヵ', 'カ'),
+ Map.entry('ㇰ', 'ク'),
+ Map.entry('ヶ', 'ケ'),
+ Map.entry('ㇱ', 'シ'),
+ Map.entry('ㇲ', 'ス'),
+ Map.entry('ッ', 'ツ'),
+ Map.entry('ㇳ', 'ト'),
+ Map.entry('ㇴ', 'ヌ'),
+ Map.entry('ㇵ', 'ハ'),
+ Map.entry('ㇶ', 'ヒ'),
+ Map.entry('ㇷ', 'フ'),
+ Map.entry('ㇸ', 'ヘ'),
+ Map.entry('ㇹ', 'ホ'),
+ Map.entry('ㇺ', 'ム'),
+ Map.entry('ャ', 'ヤ'),
+ Map.entry('ュ', 'ユ'),
+ Map.entry('ョ', 'ヨ'),
+ Map.entry('ㇻ', 'ラ'),
+ Map.entry('ㇼ', 'リ'),
+ Map.entry('ㇽ', 'ル'),
+ Map.entry('ㇾ', 'レ'),
+ Map.entry('ㇿ', 'ロ'),
+ Map.entry('ヮ', 'ワ'));
+ }
+
+ private final CharTermAttribute termAttr =
addAttribute(CharTermAttribute.class);
+
+ public JapaneseKatakanaUppercaseFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ String term = termAttr.toString();
+ // Small letter "ㇷ゚" is not single character, so it should be converted
to "プ" as String
+ term = term.replace("ㇷ゚", "プ");
+ char[] src = term.toCharArray();
Review Comment:
You could instead call `term.buffer()` to access the source `char[]` and
save creating a few temporary objects.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
