This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-codec.git
The following commit(s) were added to refs/heads/master by this push:
new c4d8365e
org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String) does
not remove special characters like punctuation
c4d8365e is described below
commit c4d8365e0fccb62dfe6a33a3054247f43be11b83
Author: Gary D. Gregory <[email protected]>
AuthorDate: Sun Jul 6 15:52:39 2025 -0400
org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String)
does not remove special characters like punctuation
---
src/changes/changes.xml | 1 +
.../codec/language/DaitchMokotoffSoundex.java | 2 +-
.../commons/codec/AbstractStringEncoderTest.java | 28 +++++++++----------
.../codec/language/DaitchMokotoffSoundexTest.java | 31 +++++++++++++++-------
4 files changed, 36 insertions(+), 26 deletions(-)
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 8d1875ec..62710c28 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -57,6 +57,7 @@ The <action> type attribute can be add,update,fix,remove.
<action type="fix" dev="ggregory" due-to="Sebastian Baunsgaard">Javadoc
typo in Base16.java #380.</action>
<action type="fix" dev="ggregory" due-to="Gary Gregory">Deprecate unused
constant org.apache.commons.codec.language.bm.Rule.ALL.</action>
<action type="fix" dev="ggregory" issue="CODEC-331" due-to="IlikeCode,
Gary
Gregory">org.apache.commons.codec.language.bm.Rule.parsePhonemeExpr(String)
adds duplicate empty phoneme when input ends with |.</action>
+ <action type="fix" dev="ggregory" issue="CODEC-331" due-to="IlikeCode,
Gary
Gregory">org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String)
does not remove special characters like punctuation.</action>
<!-- ADD -->
<action type="add" dev="ggregory" due-to="Gary Gregory">Add
HmacUtils.hmac(Path).</action>
<action type="add" dev="ggregory" due-to="Gary Gregory">Add
HmacUtils.hmacHex(Path).</action>
diff --git
a/src/main/java/org/apache/commons/codec/language/DaitchMokotoffSoundex.java
b/src/main/java/org/apache/commons/codec/language/DaitchMokotoffSoundex.java
index 1800f0d3..546a77b2 100644
--- a/src/main/java/org/apache/commons/codec/language/DaitchMokotoffSoundex.java
+++ b/src/main/java/org/apache/commons/codec/language/DaitchMokotoffSoundex.java
@@ -347,7 +347,7 @@ public class DaitchMokotoffSoundex implements StringEncoder
{
private String cleanup(final String input) {
final StringBuilder sb = new StringBuilder();
for (char ch : input.toCharArray()) {
- if (Character.isWhitespace(ch)) {
+ if (Character.isWhitespace(ch) | !Character.isLetter(ch)) {
continue;
}
ch = Character.toLowerCase(ch);
diff --git
a/src/test/java/org/apache/commons/codec/AbstractStringEncoderTest.java
b/src/test/java/org/apache/commons/codec/AbstractStringEncoderTest.java
index eabc242c..a4a9342b 100644
--- a/src/test/java/org/apache/commons/codec/AbstractStringEncoderTest.java
+++ b/src/test/java/org/apache/commons/codec/AbstractStringEncoderTest.java
@@ -29,33 +29,33 @@ import org.junit.jupiter.api.Test;
*/
public abstract class AbstractStringEncoderTest<T extends StringEncoder> {
- protected T stringEncoder = this.createStringEncoder();
+ protected T stringEncoder = createStringEncoder();
public void checkEncoding(final String expected, final String source)
throws EncoderException {
- assertEquals(expected, this.getStringEncoder().encode(source),
"Source: " + source);
+ assertEquals(expected, getStringEncoder().encode(source), "Source: " +
source);
}
protected void checkEncodings(final String[][] data) throws
EncoderException {
for (final String[] element : data) {
- this.checkEncoding(element[1], element[0]);
+ checkEncoding(element[1], element[0]);
}
}
- protected void checkEncodingVariations(final String expected, final
String[] data) throws EncoderException {
+ protected void checkEncodingVariations(final String expected, final
String... data) throws EncoderException {
for (final String element : data) {
- this.checkEncoding(expected, element);
+ checkEncoding(expected, element);
}
}
protected abstract T createStringEncoder();
public T getStringEncoder() {
- return this.stringEncoder;
+ return stringEncoder;
}
@Test
void testEncodeEmpty() throws Exception {
- final Encoder encoder = this.getStringEncoder();
+ final Encoder encoder = getStringEncoder();
encoder.encode("");
encoder.encode(" ");
encoder.encode("\t");
@@ -63,25 +63,22 @@ public abstract class AbstractStringEncoderTest<T extends
StringEncoder> {
@Test
void testEncodeNull() throws EncoderException {
- final StringEncoder encoder = this.getStringEncoder();
+ final StringEncoder encoder = getStringEncoder();
encoder.encode(null);
}
@Test
void testEncodeWithInvalidObject() throws Exception {
- final StringEncoder encoder = this.getStringEncoder();
- assertThrows(EncoderException.class, () ->
encoder.encode(Float.valueOf(3.4f)),
- "An exception was not thrown when we tried to encode a Float
object");
+ final StringEncoder encoder = getStringEncoder();
+ assertThrows(EncoderException.class, () ->
encoder.encode(Float.valueOf(3.4f)), "An exception was not thrown when we tried
to encode a Float object");
}
+
@Test
void testLocaleIndependence() throws Exception {
- final StringEncoder encoder = this.getStringEncoder();
-
+ final StringEncoder encoder = getStringEncoder();
final String[] data = { "I", "i" };
-
final Locale orig = Locale.getDefault();
final Locale[] locales = { Locale.ENGLISH, new Locale("tr"),
Locale.getDefault() };
-
try {
for (final String element : data) {
String ref = null;
@@ -104,5 +101,4 @@ public abstract class AbstractStringEncoderTest<T extends
StringEncoder> {
Locale.setDefault(orig);
}
}
-
}
diff --git
a/src/test/java/org/apache/commons/codec/language/DaitchMokotoffSoundexTest.java
b/src/test/java/org/apache/commons/codec/language/DaitchMokotoffSoundexTest.java
index 897c5b08..3ea2805c 100644
---
a/src/test/java/org/apache/commons/codec/language/DaitchMokotoffSoundexTest.java
+++
b/src/test/java/org/apache/commons/codec/language/DaitchMokotoffSoundexTest.java
@@ -14,13 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.commons.codec.language;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import java.util.stream.IntStream;
+
import org.apache.commons.codec.AbstractStringEncoderTest;
import org.apache.commons.codec.EncoderException;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
/**
* Tests {@link DaitchMokotoffSoundex}.
@@ -47,7 +52,6 @@ class DaitchMokotoffSoundexTest extends
AbstractStringEncoderTest<DaitchMokotoff
void testAccentedCharacterFolding() {
assertEquals("294795", soundex("Straßburg"));
assertEquals("294795", soundex("Strasburg"));
-
assertEquals("095600", soundex("Éregon"));
assertEquals("095600", soundex("Eregon"));
}
@@ -59,7 +63,6 @@ class DaitchMokotoffSoundexTest extends
AbstractStringEncoderTest<DaitchMokotoff
// 0-54-4---8 -> wrong
// 0-54-----8 -> correct
assertEquals("054800", soundex("AKSSOL"));
-
// GERSCHFELD
// G-E-RS-CH-F-E-L-D
// 5--4/94-5/4-7-8-3 -> wrong
@@ -82,18 +85,18 @@ class DaitchMokotoffSoundexTest extends
AbstractStringEncoderTest<DaitchMokotoff
@Test
void testEncodeIgnoreApostrophes() throws EncoderException {
- checkEncodingVariations("079600", new String[] { "OBrien", "'OBrien",
"O'Brien", "OB'rien", "OBr'ien",
- "OBri'en", "OBrie'n", "OBrien'" });
+ checkEncodingVariations("079600", "OBrien", "'OBrien", "O'Brien",
"OB'rien", "OBr'ien", "OBri'en", "OBrie'n", "OBrien'");
}
/**
* Test data from http://www.myatt.demon.co.uk/sxalg.htm
*
- * @throws EncoderException for some failure scenarios */
+ * @throws EncoderException for some failure scenarios
+ */
@Test
void testEncodeIgnoreHyphens() throws EncoderException {
- checkEncodingVariations("565463", new String[] { "KINGSMITH",
"-KINGSMITH", "K-INGSMITH", "KI-NGSMITH",
- "KIN-GSMITH", "KING-SMITH", "KINGS-MITH", "KINGSM-ITH",
"KINGSMI-TH", "KINGSMIT-H", "KINGSMITH-" });
+ checkEncodingVariations("565463", "KINGSMITH", "-KINGSMITH",
"K-INGSMITH", "KI-NGSMITH", "KIN-GSMITH", "KING-SMITH", "KINGS-MITH",
"KINGSM-ITH",
+ "KINGSMI-TH", "KINGSMIT-H", "KINGSMITH-");
}
@Test
@@ -102,6 +105,18 @@ class DaitchMokotoffSoundexTest extends
AbstractStringEncoderTest<DaitchMokotoff
assertEquals("746536", encode("Washington"));
}
+ static IntStream getNonLetters() {
+ return IntStream.rangeClosed(Character.MIN_VALUE,
Character.MAX_VALUE).filter(c -> !Character.isLetter(c));
+ }
+
+ @ParameterizedTest
+ @MethodSource("getNonLetters")
+ void testEncodeIgnoreNonLetters(final int nonLetterInt) throws
EncoderException {
+ final char nonLetterChar = (char) nonLetterInt;
+ checkEncodingVariations("746536", "Washington" + nonLetterChar,
nonLetterChar + "Washington", nonLetterChar + "Washington" + nonLetterChar,
+ "Washi" + nonLetterChar + "ngton");
+ }
+
/**
* Examples from http://www.jewishgen.org/infofiles/soundex.html
*/
@@ -116,7 +131,6 @@ class DaitchMokotoffSoundexTest extends
AbstractStringEncoderTest<DaitchMokotoff
assertEquals("370000", soundex("Topf"));
assertEquals("586660", soundex("Kleinmann"));
assertEquals("769600", soundex("Ben Aron"));
-
assertEquals("097400|097500", soundex("AUERBACH"));
assertEquals("097400|097500", soundex("OHRBACH"));
assertEquals("874400", soundex("LIPSHITZ"));
@@ -166,5 +180,4 @@ class DaitchMokotoffSoundexTest extends
AbstractStringEncoderTest<DaitchMokotoff
assertEquals("364000|464000", soundex("ţamas")); // t-cedilla
assertEquals("364000|464000", soundex("țamas")); // t-comma
}
-
}