[GitHub] [lucene] rmuir commented on issue #11976: End offset for combining characters is not incremented with ICUNormalizer2CharFilter

[GitBox]

rmuir commented on issue #11976:
URL: https://github.com/apache/lucene/issues/11976#issuecomment-1327614706

   Here is my suggested test:
   ```
   diff --git 
a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
 
b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
   index c84b8c6887e..f9febbb1d0d 100644
   --- 
a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
   +++ 
b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
   @@ -25,6 +25,8 @@ import org.apache.lucene.analysis.Analyzer;
    import org.apache.lucene.analysis.CharFilter;
    import org.apache.lucene.analysis.Tokenizer;
    import org.apache.lucene.analysis.core.KeywordTokenizer;
   +import 
org.apache.lucene.analysis.icu.segmentation.DefaultICUTokenizerConfig;
   +import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
    import org.apache.lucene.analysis.ngram.NGramTokenizer;
    import org.apache.lucene.index.IndexWriter;
    import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
   @@ -666,4 +668,23 @@ public class TestICUNormalizer2CharFilter extends 
BaseTokenStreamTestCase {
          checkAnalysisConsistency(random(), a, false, new String(text));
        }
      }
   +
   +  public void testDecomposeFromSameInputCharacter() throws IOException {
   +    String input = "日日㋀日"; // ㋀ will decompose (compatibility character)
   +    CharFilter reader =
   +        new ICUNormalizer2CharFilter(
   +            new StringReader(input),
   +            Normalizer2.getInstance(null, "nfkc_cf", 
Normalizer2.Mode.COMPOSE));
   +
   +    Tokenizer tokenStream =
   +        new ICUTokenizer(newAttributeFactory(), new 
DefaultICUTokenizerConfig(false, true));
   +    tokenStream.setReader(reader);
   +
   +    assertTokenStreamContents(
   +        tokenStream,
   +        new String[] {"日", "日", "1", "月", "日"},
   +        new int[] {0, 1, 2, 2, 3},
   +        new int[] {1, 2, 3, 3, 4},
   +        input.length());
   +  }
    }
   ```


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org