This is an automated email from the ASF dual-hosted git repository.

ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-codec.git


The following commit(s) were added to refs/heads/master by this push:
     new c4d8365e 
org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String) does 
not remove special characters like punctuation
c4d8365e is described below

commit c4d8365e0fccb62dfe6a33a3054247f43be11b83
Author: Gary D. Gregory <[email protected]>
AuthorDate: Sun Jul 6 15:52:39 2025 -0400

    org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String)
    does not remove special characters like punctuation
---
 src/changes/changes.xml                            |  1 +
 .../codec/language/DaitchMokotoffSoundex.java      |  2 +-
 .../commons/codec/AbstractStringEncoderTest.java   | 28 +++++++++----------
 .../codec/language/DaitchMokotoffSoundexTest.java  | 31 +++++++++++++++-------
 4 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 8d1875ec..62710c28 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -57,6 +57,7 @@ The <action> type attribute can be add,update,fix,remove.
       <action type="fix" dev="ggregory" due-to="Sebastian Baunsgaard">Javadoc 
typo in Base16.java #380.</action>
       <action type="fix" dev="ggregory" due-to="Gary Gregory">Deprecate unused 
constant org.apache.commons.codec.language.bm.Rule.ALL.</action>
       <action type="fix" dev="ggregory" issue="CODEC-331" due-to="IlikeCode, 
Gary 
Gregory">org.apache.commons.codec.language.bm.Rule.parsePhonemeExpr(String) 
adds duplicate empty phoneme when input ends with |.</action>
+      <action type="fix" dev="ggregory" issue="CODEC-331" due-to="IlikeCode, 
Gary 
Gregory">org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String)
 does not remove special characters like punctuation.</action>
       <!-- ADD -->
       <action type="add" dev="ggregory" due-to="Gary Gregory">Add 
HmacUtils.hmac(Path).</action>      
       <action type="add" dev="ggregory" due-to="Gary Gregory">Add 
HmacUtils.hmacHex(Path).</action>      
diff --git 
a/src/main/java/org/apache/commons/codec/language/DaitchMokotoffSoundex.java 
b/src/main/java/org/apache/commons/codec/language/DaitchMokotoffSoundex.java
index 1800f0d3..546a77b2 100644
--- a/src/main/java/org/apache/commons/codec/language/DaitchMokotoffSoundex.java
+++ b/src/main/java/org/apache/commons/codec/language/DaitchMokotoffSoundex.java
@@ -347,7 +347,7 @@ public class DaitchMokotoffSoundex implements StringEncoder 
{
     private String cleanup(final String input) {
         final StringBuilder sb = new StringBuilder();
         for (char ch : input.toCharArray()) {
-            if (Character.isWhitespace(ch)) {
+            if (Character.isWhitespace(ch) | !Character.isLetter(ch)) {
                 continue;
             }
             ch = Character.toLowerCase(ch);
diff --git 
a/src/test/java/org/apache/commons/codec/AbstractStringEncoderTest.java 
b/src/test/java/org/apache/commons/codec/AbstractStringEncoderTest.java
index eabc242c..a4a9342b 100644
--- a/src/test/java/org/apache/commons/codec/AbstractStringEncoderTest.java
+++ b/src/test/java/org/apache/commons/codec/AbstractStringEncoderTest.java
@@ -29,33 +29,33 @@ import org.junit.jupiter.api.Test;
  */
 public abstract class AbstractStringEncoderTest<T extends StringEncoder> {
 
-    protected T stringEncoder = this.createStringEncoder();
+    protected T stringEncoder = createStringEncoder();
 
     public void checkEncoding(final String expected, final String source) 
throws EncoderException {
-        assertEquals(expected, this.getStringEncoder().encode(source), 
"Source: " + source);
+        assertEquals(expected, getStringEncoder().encode(source), "Source: " + 
source);
     }
 
     protected void checkEncodings(final String[][] data) throws 
EncoderException {
         for (final String[] element : data) {
-            this.checkEncoding(element[1], element[0]);
+            checkEncoding(element[1], element[0]);
         }
     }
 
-    protected void checkEncodingVariations(final String expected, final 
String[] data) throws EncoderException {
+    protected void checkEncodingVariations(final String expected, final 
String... data) throws EncoderException {
         for (final String element : data) {
-            this.checkEncoding(expected, element);
+            checkEncoding(expected, element);
         }
     }
 
     protected abstract T createStringEncoder();
 
     public T getStringEncoder() {
-        return this.stringEncoder;
+        return stringEncoder;
     }
 
     @Test
     void testEncodeEmpty() throws Exception {
-        final Encoder encoder = this.getStringEncoder();
+        final Encoder encoder = getStringEncoder();
         encoder.encode("");
         encoder.encode(" ");
         encoder.encode("\t");
@@ -63,25 +63,22 @@ public abstract class AbstractStringEncoderTest<T extends 
StringEncoder> {
 
     @Test
     void testEncodeNull() throws EncoderException {
-        final StringEncoder encoder = this.getStringEncoder();
+        final StringEncoder encoder = getStringEncoder();
         encoder.encode(null);
     }
 
     @Test
     void testEncodeWithInvalidObject() throws Exception {
-        final StringEncoder encoder = this.getStringEncoder();
-        assertThrows(EncoderException.class, () -> 
encoder.encode(Float.valueOf(3.4f)),
-                "An exception was not thrown when we tried to encode a Float 
object");
+        final StringEncoder encoder = getStringEncoder();
+        assertThrows(EncoderException.class, () -> 
encoder.encode(Float.valueOf(3.4f)), "An exception was not thrown when we tried 
to encode a Float object");
     }
+
     @Test
     void testLocaleIndependence() throws Exception {
-        final StringEncoder encoder = this.getStringEncoder();
-
+        final StringEncoder encoder = getStringEncoder();
         final String[] data = { "I", "i" };
-
         final Locale orig = Locale.getDefault();
         final Locale[] locales = { Locale.ENGLISH, new Locale("tr"), 
Locale.getDefault() };
-
         try {
             for (final String element : data) {
                 String ref = null;
@@ -104,5 +101,4 @@ public abstract class AbstractStringEncoderTest<T extends 
StringEncoder> {
             Locale.setDefault(orig);
         }
     }
-
 }
diff --git 
a/src/test/java/org/apache/commons/codec/language/DaitchMokotoffSoundexTest.java
 
b/src/test/java/org/apache/commons/codec/language/DaitchMokotoffSoundexTest.java
index 897c5b08..3ea2805c 100644
--- 
a/src/test/java/org/apache/commons/codec/language/DaitchMokotoffSoundexTest.java
+++ 
b/src/test/java/org/apache/commons/codec/language/DaitchMokotoffSoundexTest.java
@@ -14,13 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.commons.codec.language;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import java.util.stream.IntStream;
+
 import org.apache.commons.codec.AbstractStringEncoderTest;
 import org.apache.commons.codec.EncoderException;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 /**
  * Tests {@link DaitchMokotoffSoundex}.
@@ -47,7 +52,6 @@ class DaitchMokotoffSoundexTest extends 
AbstractStringEncoderTest<DaitchMokotoff
     void testAccentedCharacterFolding() {
         assertEquals("294795", soundex("Straßburg"));
         assertEquals("294795", soundex("Strasburg"));
-
         assertEquals("095600", soundex("Éregon"));
         assertEquals("095600", soundex("Eregon"));
     }
@@ -59,7 +63,6 @@ class DaitchMokotoffSoundexTest extends 
AbstractStringEncoderTest<DaitchMokotoff
         // 0-54-4---8 -> wrong
         // 0-54-----8 -> correct
         assertEquals("054800", soundex("AKSSOL"));
-
         // GERSCHFELD
         // G-E-RS-CH-F-E-L-D
         // 5--4/94-5/4-7-8-3 -> wrong
@@ -82,18 +85,18 @@ class DaitchMokotoffSoundexTest extends 
AbstractStringEncoderTest<DaitchMokotoff
 
     @Test
     void testEncodeIgnoreApostrophes() throws EncoderException {
-        checkEncodingVariations("079600", new String[] { "OBrien", "'OBrien", 
"O'Brien", "OB'rien", "OBr'ien",
-                "OBri'en", "OBrie'n", "OBrien'" });
+        checkEncodingVariations("079600", "OBrien", "'OBrien", "O'Brien", 
"OB'rien", "OBr'ien", "OBri'en", "OBrie'n", "OBrien'");
     }
 
     /**
      * Test data from http://www.myatt.demon.co.uk/sxalg.htm
      *
-     * @throws EncoderException for some failure scenarios     */
+     * @throws EncoderException for some failure scenarios
+     */
     @Test
     void testEncodeIgnoreHyphens() throws EncoderException {
-        checkEncodingVariations("565463", new String[] { "KINGSMITH", 
"-KINGSMITH", "K-INGSMITH", "KI-NGSMITH",
-                "KIN-GSMITH", "KING-SMITH", "KINGS-MITH", "KINGSM-ITH", 
"KINGSMI-TH", "KINGSMIT-H", "KINGSMITH-" });
+        checkEncodingVariations("565463", "KINGSMITH", "-KINGSMITH", 
"K-INGSMITH", "KI-NGSMITH", "KIN-GSMITH", "KING-SMITH", "KINGS-MITH", 
"KINGSM-ITH",
+                "KINGSMI-TH", "KINGSMIT-H", "KINGSMITH-");
     }
 
     @Test
@@ -102,6 +105,18 @@ class DaitchMokotoffSoundexTest extends 
AbstractStringEncoderTest<DaitchMokotoff
         assertEquals("746536", encode("Washington"));
     }
 
+    static IntStream getNonLetters() {
+        return IntStream.rangeClosed(Character.MIN_VALUE, 
Character.MAX_VALUE).filter(c -> !Character.isLetter(c));
+    }
+
+    @ParameterizedTest
+    @MethodSource("getNonLetters")
+    void testEncodeIgnoreNonLetters(final int nonLetterInt) throws 
EncoderException {
+        final char nonLetterChar = (char) nonLetterInt;
+        checkEncodingVariations("746536", "Washington" + nonLetterChar, 
nonLetterChar + "Washington", nonLetterChar + "Washington" + nonLetterChar,
+                "Washi" + nonLetterChar + "ngton");
+    }
+
     /**
      * Examples from http://www.jewishgen.org/infofiles/soundex.html
      */
@@ -116,7 +131,6 @@ class DaitchMokotoffSoundexTest extends 
AbstractStringEncoderTest<DaitchMokotoff
         assertEquals("370000", soundex("Topf"));
         assertEquals("586660", soundex("Kleinmann"));
         assertEquals("769600", soundex("Ben Aron"));
-
         assertEquals("097400|097500", soundex("AUERBACH"));
         assertEquals("097400|097500", soundex("OHRBACH"));
         assertEquals("874400", soundex("LIPSHITZ"));
@@ -166,5 +180,4 @@ class DaitchMokotoffSoundexTest extends 
AbstractStringEncoderTest<DaitchMokotoff
         assertEquals("364000|464000", soundex("ţamas")); // t-cedilla
         assertEquals("364000|464000", soundex("țamas")); // t-comma
     }
-
 }

Reply via email to