mikemccand commented on code in PR #12915:
URL: https://github.com/apache/lucene/pull/12915#discussion_r1424520399
##########
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseHiraganaUppercaseFilter.java:
##########
@@ -0,0 +1,65 @@
+package org.apache.lucene.analysis.ja;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/**
+ * A {@link TokenFilter} that normalizes small letters (捨て仮名) in hiragana into
normal letters. For
+ * instance, "ちょっとまって" will be translated to "ちよつとまつて".
+ *
+ * <p>This filter is useful if you want to search against old style Japanese
text such as patents,
+ * legal, contract policies, etc.
+ */
+public final class JapaneseHiraganaUppercaseFilter extends TokenFilter {
+ private static final Map<Character, Character> s2l;
+
+ static {
+ // supported characters are:
+ // ぁ ぃ ぅ ぇ ぉ っ ゃ ゅ ょ ゎ ゕ ゖ
+ s2l =
+ Map.ofEntries(
+ Map.entry('ぁ', 'あ'),
+ Map.entry('ぃ', 'い'),
+ Map.entry('ぅ', 'う'),
+ Map.entry('ぇ', 'え'),
+ Map.entry('ぉ', 'お'),
+ Map.entry('っ', 'つ'),
+ Map.entry('ゃ', 'や'),
+ Map.entry('ゅ', 'ゆ'),
+ Map.entry('ょ', 'よ'),
+ Map.entry('ゎ', 'わ'),
+ Map.entry('ゕ', 'か'),
+ Map.entry('ゖ', 'け'));
+ }
+
+ private final CharTermAttribute termAttr =
addAttribute(CharTermAttribute.class);
+
+ public JapaneseHiraganaUppercaseFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ String term = termAttr.toString();
+ char[] src = term.toCharArray();
+ char[] result = new char[src.length];
+ for (int i = 0; i < src.length; i++) {
+ Character c = s2l.get(src[i]);
+ if (c != null) {
+ result[i] = c;
+ } else {
+ result[i] = src[i];
+ }
+ }
+ String resultTerm = String.copyValueOf(result);
+ termAttr.setEmpty().append(resultTerm);
Review Comment:
> This will eliminate all of the byte copy. I don't know if we are supposed
to do that (but the API allow). Maybe @mikemccand could have some thought here.
This is indeed the intended usage for high performance -- directly alter
that underlying `char[]` buffer, asking the term att to grow if needed, and
setting the length when you are done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
