Repository: commons-text Updated Branches: refs/heads/master c8a803c21 -> ad7ee397a
TEXT-23: Moving over o.a.c.l.text.translate, no tests Project: http://git-wip-us.apache.org/repos/asf/commons-text/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-text/commit/617869b2 Tree: http://git-wip-us.apache.org/repos/asf/commons-text/tree/617869b2 Diff: http://git-wip-us.apache.org/repos/asf/commons-text/diff/617869b2 Branch: refs/heads/master Commit: 617869b228757d41086b08730fb31abbb61f4b83 Parents: a0cf833 Author: Rob Tompkins <chtom...@gmail.com> Authored: Mon Nov 7 15:05:17 2016 -0500 Committer: Rob Tompkins <chtom...@gmail.com> Committed: Mon Nov 7 15:05:17 2016 -0500 ---------------------------------------------------------------------- .../text/translate/AggregateTranslator.java | 59 +++ .../text/translate/CharSequenceTranslator.java | 135 ++++++ .../text/translate/CodePointTranslator.java | 51 +++ .../commons/text/translate/EntityArrays.java | 424 +++++++++++++++++++ .../text/translate/JavaUnicodeEscaper.java | 113 +++++ .../text/translate/LookupTranslator.java | 91 ++++ .../text/translate/NumericEntityEscaper.java | 118 ++++++ .../text/translate/NumericEntityUnescaper.java | 138 ++++++ .../commons/text/translate/OctalUnescaper.java | 79 ++++ .../commons/text/translate/UnicodeEscaper.java | 138 ++++++ .../text/translate/UnicodeUnescaper.java | 64 +++ .../UnicodeUnpairedSurrogateRemover.java | 40 ++ .../commons/text/translate/package-info.java | 24 ++ 13 files changed, 1474 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/AggregateTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/AggregateTranslator.java b/src/main/java/org/apache/commons/text/translate/AggregateTranslator.java new file mode 100644 index 0000000..d550037 --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/AggregateTranslator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import org.apache.commons.lang3.ArrayUtils; + +import java.io.IOException; +import java.io.Writer; + +/** + * Executes a sequence of translators one after the other. Execution ends whenever + * the first translator consumes codepoints from the input. + * + * @since 3.0 + */ +public class AggregateTranslator extends CharSequenceTranslator { + + private final CharSequenceTranslator[] translators; + + /** + * Specify the translators to be used at creation time. + * + * @param translators CharSequenceTranslator array to aggregate + */ + public AggregateTranslator(final CharSequenceTranslator... translators) { + this.translators = ArrayUtils.clone(translators); + } + + /** + * The first translator to consume codepoints from the input is the 'winner'. + * Execution stops with the number of consumed codepoints being returned. + * {@inheritDoc} + */ + @Override + public int translate(final CharSequence input, final int index, final Writer out) throws IOException { + for (final CharSequenceTranslator translator : translators) { + final int consumed = translator.translate(input, index, out); + if(consumed != 0) { + return consumed; + } + } + return 0; + } + +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/CharSequenceTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/CharSequenceTranslator.java b/src/main/java/org/apache/commons/text/translate/CharSequenceTranslator.java new file mode 100644 index 0000000..16ec02f --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/CharSequenceTranslator.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.Locale; + +/** + * An API for translating text. + * Its core use is to escape and unescape text. Because escaping and unescaping + * is completely contextual, the API does not present two separate signatures. + * + * @since 3.0 + */ +public abstract class CharSequenceTranslator { + + static final char[] HEX_DIGITS = new char[] {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; + + /** + * Translate a set of codepoints, represented by an int index into a CharSequence, + * into another set of codepoints. The number of codepoints consumed must be returned, + * and the only IOExceptions thrown must be from interacting with the Writer so that + * the top level API may reliably ignore StringWriter IOExceptions. + * + * @param input CharSequence that is being translated + * @param index int representing the current point of translation + * @param out Writer to translate the text to + * @return int count of codepoints consumed + * @throws IOException if and only if the Writer produces an IOException + */ + public abstract int translate(CharSequence input, int index, Writer out) throws IOException; + + /** + * Helper for non-Writer usage. + * @param input CharSequence to be translated + * @return String output of translation + */ + public final String translate(final CharSequence input) { + if (input == null) { + return null; + } + try { + final StringWriter writer = new StringWriter(input.length() * 2); + translate(input, writer); + return writer.toString(); + } catch (final IOException ioe) { + // this should never ever happen while writing to a StringWriter + throw new RuntimeException(ioe); + } + } + + /** + * Translate an input onto a Writer. This is intentionally final as its algorithm is + * tightly coupled with the abstract method of this class. + * + * @param input CharSequence that is being translated + * @param out Writer to translate the text to + * @throws IOException if and only if the Writer produces an IOException + */ + public final void translate(final CharSequence input, final Writer out) throws IOException { + if (out == null) { + throw new IllegalArgumentException("The Writer must not be null"); + } + if (input == null) { + return; + } + int pos = 0; + final int len = input.length(); + while (pos < len) { + final int consumed = translate(input, pos, out); + if (consumed == 0) { + // inlined implementation of Character.toChars(Character.codePointAt(input, pos)) + // avoids allocating temp char arrays and duplicate checks + char c1 = input.charAt(pos); + out.write(c1); + pos++; + if (Character.isHighSurrogate(c1) && pos < len) { + char c2 = input.charAt(pos); + if (Character.isLowSurrogate(c2)) { + out.write(c2); + pos++; + } + } + continue; + } + // contract with translators is that they have to understand codepoints + // and they just took care of a surrogate pair + for (int pt = 0; pt < consumed; pt++) { + pos += Character.charCount(Character.codePointAt(input, pos)); + } + } + } + + /** + * Helper method to create a merger of this translator with another set of + * translators. Useful in customizing the standard functionality. + * + * @param translators CharSequenceTranslator array of translators to merge with this one + * @return CharSequenceTranslator merging this translator with the others + */ + public final org.apache.commons.text.translate.CharSequenceTranslator with(final org.apache.commons.text.translate.CharSequenceTranslator... translators) { + final org.apache.commons.text.translate.CharSequenceTranslator[] newArray = new org.apache.commons.text.translate.CharSequenceTranslator[translators.length + 1]; + newArray[0] = this; + System.arraycopy(translators, 0, newArray, 1, translators.length); + return new AggregateTranslator(newArray); + } + + /** + * <p>Returns an upper case hexadecimal <code>String</code> for the given + * character.</p> + * + * @param codepoint The codepoint to convert. + * @return An upper case hexadecimal <code>String</code> + */ + public static String hex(final int codepoint) { + return Integer.toHexString(codepoint).toUpperCase(Locale.ENGLISH); + } + +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/CodePointTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/CodePointTranslator.java b/src/main/java/org/apache/commons/text/translate/CodePointTranslator.java new file mode 100644 index 0000000..cac3d8f --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/CodePointTranslator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import java.io.IOException; +import java.io.Writer; + +/** + * Helper subclass to CharSequenceTranslator to allow for translations that + * will replace up to one character at a time. + * + * @since 3.0 + */ +public abstract class CodePointTranslator extends CharSequenceTranslator { + + /** + * Implementation of translate that maps onto the abstract translate(int, Writer) method. + * {@inheritDoc} + */ + @Override + public final int translate(final CharSequence input, final int index, final Writer out) throws IOException { + final int codepoint = Character.codePointAt(input, index); + final boolean consumed = translate(codepoint, out); + return consumed ? 1 : 0; + } + + /** + * Translate the specified codepoint into another. + * + * @param codepoint int character input to translate + * @param out Writer to optionally push the translated output to + * @return boolean as to whether translation occurred or not + * @throws IOException if and only if the Writer produces an IOException + */ + public abstract boolean translate(int codepoint, Writer out) throws IOException; + +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/EntityArrays.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/EntityArrays.java b/src/main/java/org/apache/commons/text/translate/EntityArrays.java new file mode 100644 index 0000000..5c7c4e3 --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/EntityArrays.java @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +/** + * Class holding various entity data for HTML and XML - generally for use with + * the LookupTranslator. + * All arrays are of length [*][2]. + * + * @since 3.0 + */ +public class EntityArrays { + + /** + * Mapping to escape <a href="https://secure.wikimedia.org/wikipedia/en/wiki/ISO/IEC_8859-1";>ISO-8859-1</a> + * characters to their named HTML 3.x equivalents. + * @return the mapping table + */ + public static String[][] ISO8859_1_ESCAPE() { return ISO8859_1_ESCAPE.clone(); } + private static final String[][] ISO8859_1_ESCAPE = { + {"\u00A0", " "}, // non-breaking space + {"\u00A1", "¡"}, // inverted exclamation mark + {"\u00A2", "¢"}, // cent sign + {"\u00A3", "£"}, // pound sign + {"\u00A4", "¤"}, // currency sign + {"\u00A5", "¥"}, // yen sign = yuan sign + {"\u00A6", "¦"}, // broken bar = broken vertical bar + {"\u00A7", "§"}, // section sign + {"\u00A8", "¨"}, // diaeresis = spacing diaeresis + {"\u00A9", "©"}, // © - copyright sign + {"\u00AA", "ª"}, // feminine ordinal indicator + {"\u00AB", "«"}, // left-pointing double angle quotation mark = left pointing guillemet + {"\u00AC", "¬"}, // not sign + {"\u00AD", "­"}, // soft hyphen = discretionary hyphen + {"\u00AE", "®"}, // ® - registered trademark sign + {"\u00AF", "¯"}, // macron = spacing macron = overline = APL overbar + {"\u00B0", "°"}, // degree sign + {"\u00B1", "±"}, // plus-minus sign = plus-or-minus sign + {"\u00B2", "²"}, // superscript two = superscript digit two = squared + {"\u00B3", "³"}, // superscript three = superscript digit three = cubed + {"\u00B4", "´"}, // acute accent = spacing acute + {"\u00B5", "µ"}, // micro sign + {"\u00B6", "¶"}, // pilcrow sign = paragraph sign + {"\u00B7", "·"}, // middle dot = Georgian comma = Greek middle dot + {"\u00B8", "¸"}, // cedilla = spacing cedilla + {"\u00B9", "¹"}, // superscript one = superscript digit one + {"\u00BA", "º"}, // masculine ordinal indicator + {"\u00BB", "»"}, // right-pointing double angle quotation mark = right pointing guillemet + {"\u00BC", "¼"}, // vulgar fraction one quarter = fraction one quarter + {"\u00BD", "½"}, // vulgar fraction one half = fraction one half + {"\u00BE", "¾"}, // vulgar fraction three quarters = fraction three quarters + {"\u00BF", "¿"}, // inverted question mark = turned question mark + {"\u00C0", "À"}, // à - uppercase A, grave accent + {"\u00C1", "Á"}, // à - uppercase A, acute accent + {"\u00C2", "Â"}, // à - uppercase A, circumflex accent + {"\u00C3", "Ã"}, // à - uppercase A, tilde + {"\u00C4", "Ä"}, // à - uppercase A, umlaut + {"\u00C5", "Å"}, // à - uppercase A, ring + {"\u00C6", "Æ"}, // à - uppercase AE + {"\u00C7", "Ç"}, // à - uppercase C, cedilla + {"\u00C8", "È"}, // à - uppercase E, grave accent + {"\u00C9", "É"}, // à - uppercase E, acute accent + {"\u00CA", "Ê"}, // à - uppercase E, circumflex accent + {"\u00CB", "Ë"}, // à - uppercase E, umlaut + {"\u00CC", "Ì"}, // à - uppercase I, grave accent + {"\u00CD", "Í"}, // à - uppercase I, acute accent + {"\u00CE", "Î"}, // à - uppercase I, circumflex accent + {"\u00CF", "Ï"}, // à - uppercase I, umlaut + {"\u00D0", "Ð"}, // à - uppercase Eth, Icelandic + {"\u00D1", "Ñ"}, // à - uppercase N, tilde + {"\u00D2", "Ò"}, // à - uppercase O, grave accent + {"\u00D3", "Ó"}, // à - uppercase O, acute accent + {"\u00D4", "Ô"}, // à - uppercase O, circumflex accent + {"\u00D5", "Õ"}, // à - uppercase O, tilde + {"\u00D6", "Ö"}, // à - uppercase O, umlaut + {"\u00D7", "×"}, // multiplication sign + {"\u00D8", "Ø"}, // à - uppercase O, slash + {"\u00D9", "Ù"}, // à - uppercase U, grave accent + {"\u00DA", "Ú"}, // à - uppercase U, acute accent + {"\u00DB", "Û"}, // à - uppercase U, circumflex accent + {"\u00DC", "Ü"}, // à - uppercase U, umlaut + {"\u00DD", "Ý"}, // à - uppercase Y, acute accent + {"\u00DE", "Þ"}, // à - uppercase THORN, Icelandic + {"\u00DF", "ß"}, // à - lowercase sharps, German + {"\u00E0", "à"}, // à - lowercase a, grave accent + {"\u00E1", "á"}, // á - lowercase a, acute accent + {"\u00E2", "â"}, // â - lowercase a, circumflex accent + {"\u00E3", "ã"}, // ã - lowercase a, tilde + {"\u00E4", "ä"}, // ä - lowercase a, umlaut + {"\u00E5", "å"}, // Ã¥ - lowercase a, ring + {"\u00E6", "æ"}, // æ - lowercase ae + {"\u00E7", "ç"}, // ç - lowercase c, cedilla + {"\u00E8", "è"}, // è - lowercase e, grave accent + {"\u00E9", "é"}, // é - lowercase e, acute accent + {"\u00EA", "ê"}, // ê - lowercase e, circumflex accent + {"\u00EB", "ë"}, // ë - lowercase e, umlaut + {"\u00EC", "ì"}, // ì - lowercase i, grave accent + {"\u00ED", "í"}, // à - lowercase i, acute accent + {"\u00EE", "î"}, // î - lowercase i, circumflex accent + {"\u00EF", "ï"}, // ï - lowercase i, umlaut + {"\u00F0", "ð"}, // ð - lowercase eth, Icelandic + {"\u00F1", "ñ"}, // ñ - lowercase n, tilde + {"\u00F2", "ò"}, // ò - lowercase o, grave accent + {"\u00F3", "ó"}, // ó - lowercase o, acute accent + {"\u00F4", "ô"}, // ô - lowercase o, circumflex accent + {"\u00F5", "õ"}, // õ - lowercase o, tilde + {"\u00F6", "ö"}, // ö - lowercase o, umlaut + {"\u00F7", "÷"}, // division sign + {"\u00F8", "ø"}, // ø - lowercase o, slash + {"\u00F9", "ù"}, // ù - lowercase u, grave accent + {"\u00FA", "ú"}, // ú - lowercase u, acute accent + {"\u00FB", "û"}, // û - lowercase u, circumflex accent + {"\u00FC", "ü"}, // ü - lowercase u, umlaut + {"\u00FD", "ý"}, // ý - lowercase y, acute accent + {"\u00FE", "þ"}, // þ - lowercase thorn, Icelandic + {"\u00FF", "ÿ"}, // ÿ - lowercase y, umlaut + }; + + /** + * Reverse of {@link #ISO8859_1_ESCAPE()} for unescaping purposes. + * @return the mapping table + */ + public static String[][] ISO8859_1_UNESCAPE() { return ISO8859_1_UNESCAPE.clone(); } + private static final String[][] ISO8859_1_UNESCAPE = invert(ISO8859_1_ESCAPE); + + /** + * Mapping to escape additional <a href="http://www.w3.org/TR/REC-html40/sgml/entities.html";>character entity + * references</a>. Note that this must be used with {@link #ISO8859_1_ESCAPE()} to get the full list of + * HTML 4.0 character entities. + * @return the mapping table + */ + public static String[][] HTML40_EXTENDED_ESCAPE() { return HTML40_EXTENDED_ESCAPE.clone(); } + private static final String[][] HTML40_EXTENDED_ESCAPE = { + // <!-- Latin Extended-B --> + {"\u0192", "ƒ"}, // latin small f with hook = function= florin, U+0192 ISOtech --> + // <!-- Greek --> + {"\u0391", "Α"}, // greek capital letter alpha, U+0391 --> + {"\u0392", "Β"}, // greek capital letter beta, U+0392 --> + {"\u0393", "Γ"}, // greek capital letter gamma,U+0393 ISOgrk3 --> + {"\u0394", "Δ"}, // greek capital letter delta,U+0394 ISOgrk3 --> + {"\u0395", "Ε"}, // greek capital letter epsilon, U+0395 --> + {"\u0396", "Ζ"}, // greek capital letter zeta, U+0396 --> + {"\u0397", "Η"}, // greek capital letter eta, U+0397 --> + {"\u0398", "Θ"}, // greek capital letter theta,U+0398 ISOgrk3 --> + {"\u0399", "Ι"}, // greek capital letter iota, U+0399 --> + {"\u039A", "Κ"}, // greek capital letter kappa, U+039A --> + {"\u039B", "Λ"}, // greek capital letter lambda,U+039B ISOgrk3 --> + {"\u039C", "Μ"}, // greek capital letter mu, U+039C --> + {"\u039D", "Ν"}, // greek capital letter nu, U+039D --> + {"\u039E", "Ξ"}, // greek capital letter xi, U+039E ISOgrk3 --> + {"\u039F", "Ο"}, // greek capital letter omicron, U+039F --> + {"\u03A0", "Π"}, // greek capital letter pi, U+03A0 ISOgrk3 --> + {"\u03A1", "Ρ"}, // greek capital letter rho, U+03A1 --> + // <!-- there is no Sigmaf, and no U+03A2 character either --> + {"\u03A3", "Σ"}, // greek capital letter sigma,U+03A3 ISOgrk3 --> + {"\u03A4", "Τ"}, // greek capital letter tau, U+03A4 --> + {"\u03A5", "Υ"}, // greek capital letter upsilon,U+03A5 ISOgrk3 --> + {"\u03A6", "Φ"}, // greek capital letter phi,U+03A6 ISOgrk3 --> + {"\u03A7", "Χ"}, // greek capital letter chi, U+03A7 --> + {"\u03A8", "Ψ"}, // greek capital letter psi,U+03A8 ISOgrk3 --> + {"\u03A9", "Ω"}, // greek capital letter omega,U+03A9 ISOgrk3 --> + {"\u03B1", "α"}, // greek small letter alpha,U+03B1 ISOgrk3 --> + {"\u03B2", "β"}, // greek small letter beta, U+03B2 ISOgrk3 --> + {"\u03B3", "γ"}, // greek small letter gamma,U+03B3 ISOgrk3 --> + {"\u03B4", "δ"}, // greek small letter delta,U+03B4 ISOgrk3 --> + {"\u03B5", "ε"}, // greek small letter epsilon,U+03B5 ISOgrk3 --> + {"\u03B6", "ζ"}, // greek small letter zeta, U+03B6 ISOgrk3 --> + {"\u03B7", "η"}, // greek small letter eta, U+03B7 ISOgrk3 --> + {"\u03B8", "θ"}, // greek small letter theta,U+03B8 ISOgrk3 --> + {"\u03B9", "ι"}, // greek small letter iota, U+03B9 ISOgrk3 --> + {"\u03BA", "κ"}, // greek small letter kappa,U+03BA ISOgrk3 --> + {"\u03BB", "λ"}, // greek small letter lambda,U+03BB ISOgrk3 --> + {"\u03BC", "μ"}, // greek small letter mu, U+03BC ISOgrk3 --> + {"\u03BD", "ν"}, // greek small letter nu, U+03BD ISOgrk3 --> + {"\u03BE", "ξ"}, // greek small letter xi, U+03BE ISOgrk3 --> + {"\u03BF", "ο"}, // greek small letter omicron, U+03BF NEW --> + {"\u03C0", "π"}, // greek small letter pi, U+03C0 ISOgrk3 --> + {"\u03C1", "ρ"}, // greek small letter rho, U+03C1 ISOgrk3 --> + {"\u03C2", "ς"}, // greek small letter final sigma,U+03C2 ISOgrk3 --> + {"\u03C3", "σ"}, // greek small letter sigma,U+03C3 ISOgrk3 --> + {"\u03C4", "τ"}, // greek small letter tau, U+03C4 ISOgrk3 --> + {"\u03C5", "υ"}, // greek small letter upsilon,U+03C5 ISOgrk3 --> + {"\u03C6", "φ"}, // greek small letter phi, U+03C6 ISOgrk3 --> + {"\u03C7", "χ"}, // greek small letter chi, U+03C7 ISOgrk3 --> + {"\u03C8", "ψ"}, // greek small letter psi, U+03C8 ISOgrk3 --> + {"\u03C9", "ω"}, // greek small letter omega,U+03C9 ISOgrk3 --> + {"\u03D1", "ϑ"}, // greek small letter theta symbol,U+03D1 NEW --> + {"\u03D2", "ϒ"}, // greek upsilon with hook symbol,U+03D2 NEW --> + {"\u03D6", "ϖ"}, // greek pi symbol, U+03D6 ISOgrk3 --> + // <!-- General Punctuation --> + {"\u2022", "•"}, // bullet = black small circle,U+2022 ISOpub --> + // <!-- bullet is NOT the same as bullet operator, U+2219 --> + {"\u2026", "…"}, // horizontal ellipsis = three dot leader,U+2026 ISOpub --> + {"\u2032", "′"}, // prime = minutes = feet, U+2032 ISOtech --> + {"\u2033", "″"}, // double prime = seconds = inches,U+2033 ISOtech --> + {"\u203E", "‾"}, // overline = spacing overscore,U+203E NEW --> + {"\u2044", "⁄"}, // fraction slash, U+2044 NEW --> + // <!-- Letterlike Symbols --> + {"\u2118", "℘"}, // script capital P = power set= Weierstrass p, U+2118 ISOamso --> + {"\u2111", "ℑ"}, // blackletter capital I = imaginary part,U+2111 ISOamso --> + {"\u211C", "ℜ"}, // blackletter capital R = real part symbol,U+211C ISOamso --> + {"\u2122", "™"}, // trade mark sign, U+2122 ISOnum --> + {"\u2135", "ℵ"}, // alef symbol = first transfinite cardinal,U+2135 NEW --> + // <!-- alef symbol is NOT the same as hebrew letter alef,U+05D0 although the + // same glyph could be used to depict both characters --> + // <!-- Arrows --> + {"\u2190", "←"}, // leftwards arrow, U+2190 ISOnum --> + {"\u2191", "↑"}, // upwards arrow, U+2191 ISOnum--> + {"\u2192", "→"}, // rightwards arrow, U+2192 ISOnum --> + {"\u2193", "↓"}, // downwards arrow, U+2193 ISOnum --> + {"\u2194", "↔"}, // left right arrow, U+2194 ISOamsa --> + {"\u21B5", "↵"}, // downwards arrow with corner leftwards= carriage return, U+21B5 NEW --> + {"\u21D0", "⇐"}, // leftwards double arrow, U+21D0 ISOtech --> + // <!-- ISO 10646 does not say that lArr is the same as the 'is implied by' + // arrow but also does not have any other character for that function. + // So ? lArr canbe used for 'is implied by' as ISOtech suggests --> + {"\u21D1", "⇑"}, // upwards double arrow, U+21D1 ISOamsa --> + {"\u21D2", "⇒"}, // rightwards double arrow,U+21D2 ISOtech --> + // <!-- ISO 10646 does not say this is the 'implies' character but does not + // have another character with this function so ?rArr can be used for + // 'implies' as ISOtech suggests --> + {"\u21D3", "⇓"}, // downwards double arrow, U+21D3 ISOamsa --> + {"\u21D4", "⇔"}, // left right double arrow,U+21D4 ISOamsa --> + // <!-- Mathematical Operators --> + {"\u2200", "∀"}, // for all, U+2200 ISOtech --> + {"\u2202", "∂"}, // partial differential, U+2202 ISOtech --> + {"\u2203", "∃"}, // there exists, U+2203 ISOtech --> + {"\u2205", "∅"}, // empty set = null set = diameter,U+2205 ISOamso --> + {"\u2207", "∇"}, // nabla = backward difference,U+2207 ISOtech --> + {"\u2208", "∈"}, // element of, U+2208 ISOtech --> + {"\u2209", "∉"}, // not an element of, U+2209 ISOtech --> + {"\u220B", "∋"}, // contains as member, U+220B ISOtech --> + // <!-- should there be a more memorable name than 'ni'? --> + {"\u220F", "∏"}, // n-ary product = product sign,U+220F ISOamsb --> + // <!-- prod is NOT the same character as U+03A0 'greek capital letter pi' + // though the same glyph might be used for both --> + {"\u2211", "∑"}, // n-ary summation, U+2211 ISOamsb --> + // <!-- sum is NOT the same character as U+03A3 'greek capital letter sigma' + // though the same glyph might be used for both --> + {"\u2212", "−"}, // minus sign, U+2212 ISOtech --> + {"\u2217", "∗"}, // asterisk operator, U+2217 ISOtech --> + {"\u221A", "√"}, // square root = radical sign,U+221A ISOtech --> + {"\u221D", "∝"}, // proportional to, U+221D ISOtech --> + {"\u221E", "∞"}, // infinity, U+221E ISOtech --> + {"\u2220", "∠"}, // angle, U+2220 ISOamso --> + {"\u2227", "∧"}, // logical and = wedge, U+2227 ISOtech --> + {"\u2228", "∨"}, // logical or = vee, U+2228 ISOtech --> + {"\u2229", "∩"}, // intersection = cap, U+2229 ISOtech --> + {"\u222A", "∪"}, // union = cup, U+222A ISOtech --> + {"\u222B", "∫"}, // integral, U+222B ISOtech --> + {"\u2234", "∴"}, // therefore, U+2234 ISOtech --> + {"\u223C", "∼"}, // tilde operator = varies with = similar to,U+223C ISOtech --> + // <!-- tilde operator is NOT the same character as the tilde, U+007E,although + // the same glyph might be used to represent both --> + {"\u2245", "≅"}, // approximately equal to, U+2245 ISOtech --> + {"\u2248", "≈"}, // almost equal to = asymptotic to,U+2248 ISOamsr --> + {"\u2260", "≠"}, // not equal to, U+2260 ISOtech --> + {"\u2261", "≡"}, // identical to, U+2261 ISOtech --> + {"\u2264", "≤"}, // less-than or equal to, U+2264 ISOtech --> + {"\u2265", "≥"}, // greater-than or equal to,U+2265 ISOtech --> + {"\u2282", "⊂"}, // subset of, U+2282 ISOtech --> + {"\u2283", "⊃"}, // superset of, U+2283 ISOtech --> + // <!-- note that nsup, 'not a superset of, U+2283' is not covered by the + // Symbol font encoding and is not included. Should it be, for symmetry? + // It is in ISOamsn -->, + {"\u2284", "⊄"}, // not a subset of, U+2284 ISOamsn --> + {"\u2286", "⊆"}, // subset of or equal to, U+2286 ISOtech --> + {"\u2287", "⊇"}, // superset of or equal to,U+2287 ISOtech --> + {"\u2295", "⊕"}, // circled plus = direct sum,U+2295 ISOamsb --> + {"\u2297", "⊗"}, // circled times = vector product,U+2297 ISOamsb --> + {"\u22A5", "⊥"}, // up tack = orthogonal to = perpendicular,U+22A5 ISOtech --> + {"\u22C5", "⋅"}, // dot operator, U+22C5 ISOamsb --> + // <!-- dot operator is NOT the same character as U+00B7 middle dot --> + // <!-- Miscellaneous Technical --> + {"\u2308", "⌈"}, // left ceiling = apl upstile,U+2308 ISOamsc --> + {"\u2309", "⌉"}, // right ceiling, U+2309 ISOamsc --> + {"\u230A", "⌊"}, // left floor = apl downstile,U+230A ISOamsc --> + {"\u230B", "⌋"}, // right floor, U+230B ISOamsc --> + {"\u2329", "⟨"}, // left-pointing angle bracket = bra,U+2329 ISOtech --> + // <!-- lang is NOT the same character as U+003C 'less than' or U+2039 'single left-pointing angle quotation + // mark' --> + {"\u232A", "⟩"}, // right-pointing angle bracket = ket,U+232A ISOtech --> + // <!-- rang is NOT the same character as U+003E 'greater than' or U+203A + // 'single right-pointing angle quotation mark' --> + // <!-- Geometric Shapes --> + {"\u25CA", "◊"}, // lozenge, U+25CA ISOpub --> + // <!-- Miscellaneous Symbols --> + {"\u2660", "♠"}, // black spade suit, U+2660 ISOpub --> + // <!-- black here seems to mean filled as opposed to hollow --> + {"\u2663", "♣"}, // black club suit = shamrock,U+2663 ISOpub --> + {"\u2665", "♥"}, // black heart suit = valentine,U+2665 ISOpub --> + {"\u2666", "♦"}, // black diamond suit, U+2666 ISOpub --> + + // <!-- Latin Extended-A --> + {"\u0152", "Œ"}, // -- latin capital ligature OE,U+0152 ISOlat2 --> + {"\u0153", "œ"}, // -- latin small ligature oe, U+0153 ISOlat2 --> + // <!-- ligature is a misnomer, this is a separate character in some languages --> + {"\u0160", "Š"}, // -- latin capital letter S with caron,U+0160 ISOlat2 --> + {"\u0161", "š"}, // -- latin small letter s with caron,U+0161 ISOlat2 --> + {"\u0178", "Ÿ"}, // -- latin capital letter Y with diaeresis,U+0178 ISOlat2 --> + // <!-- Spacing Modifier Letters --> + {"\u02C6", "ˆ"}, // -- modifier letter circumflex accent,U+02C6 ISOpub --> + {"\u02DC", "˜"}, // small tilde, U+02DC ISOdia --> + // <!-- General Punctuation --> + {"\u2002", " "}, // en space, U+2002 ISOpub --> + {"\u2003", " "}, // em space, U+2003 ISOpub --> + {"\u2009", " "}, // thin space, U+2009 ISOpub --> + {"\u200C", "‌"}, // zero width non-joiner,U+200C NEW RFC 2070 --> + {"\u200D", "‍"}, // zero width joiner, U+200D NEW RFC 2070 --> + {"\u200E", "‎"}, // left-to-right mark, U+200E NEW RFC 2070 --> + {"\u200F", "‏"}, // right-to-left mark, U+200F NEW RFC 2070 --> + {"\u2013", "–"}, // en dash, U+2013 ISOpub --> + {"\u2014", "—"}, // em dash, U+2014 ISOpub --> + {"\u2018", "‘"}, // left single quotation mark,U+2018 ISOnum --> + {"\u2019", "’"}, // right single quotation mark,U+2019 ISOnum --> + {"\u201A", "‚"}, // single low-9 quotation mark, U+201A NEW --> + {"\u201C", "“"}, // left double quotation mark,U+201C ISOnum --> + {"\u201D", "”"}, // right double quotation mark,U+201D ISOnum --> + {"\u201E", "„"}, // double low-9 quotation mark, U+201E NEW --> + {"\u2020", "†"}, // dagger, U+2020 ISOpub --> + {"\u2021", "‡"}, // double dagger, U+2021 ISOpub --> + {"\u2030", "‰"}, // per mille sign, U+2030 ISOtech --> + {"\u2039", "‹"}, // single left-pointing angle quotation mark,U+2039 ISO proposed --> + // <!-- lsaquo is proposed but not yet ISO standardized --> + {"\u203A", "›"}, // single right-pointing angle quotation mark,U+203A ISO proposed --> + // <!-- rsaquo is proposed but not yet ISO standardized --> + {"\u20AC", "€"}, // -- euro sign, U+20AC NEW --> + }; + + /** + * Reverse of {@link #HTML40_EXTENDED_ESCAPE()} for unescaping purposes. + * @return the mapping table + */ + public static String[][] HTML40_EXTENDED_UNESCAPE() { return HTML40_EXTENDED_UNESCAPE.clone(); } + private static final String[][] HTML40_EXTENDED_UNESCAPE = invert(HTML40_EXTENDED_ESCAPE); + + /** + * Mapping to escape the basic XML and HTML character entities. + * + * Namely: {@code " & < >} + * @return the mapping table + */ + public static String[][] BASIC_ESCAPE() { return BASIC_ESCAPE.clone(); } + private static final String[][] BASIC_ESCAPE = { + {"\"", """}, // " - double-quote + {"&", "&"}, // & - ampersand + {"<", "<"}, // < - less-than + {">", ">"}, // > - greater-than + }; + + /** + * Reverse of {@link #BASIC_ESCAPE()} for unescaping purposes. + * @return the mapping table + */ + public static String[][] BASIC_UNESCAPE() { return BASIC_UNESCAPE.clone(); } + private static final String[][] BASIC_UNESCAPE = invert(BASIC_ESCAPE); + + /** + * Mapping to escape the apostrophe character to its XML character entity. + * @return the mapping table + */ + public static String[][] APOS_ESCAPE() { return APOS_ESCAPE.clone(); } + private static final String[][] APOS_ESCAPE = { + {"'", "'"}, // XML apostrophe + }; + + /** + * Reverse of {@link #APOS_ESCAPE()} for unescaping purposes. + * @return the mapping table + */ + public static String[][] APOS_UNESCAPE() { return APOS_UNESCAPE.clone(); } + private static final String[][] APOS_UNESCAPE = invert(APOS_ESCAPE); + + /** + * Mapping to escape the Java control characters. + * + * Namely: {@code \b \n \t \f \r} + * @return the mapping table + */ + public static String[][] JAVA_CTRL_CHARS_ESCAPE() { return JAVA_CTRL_CHARS_ESCAPE.clone(); } + private static final String[][] JAVA_CTRL_CHARS_ESCAPE = { + {"\b", "\\b"}, + {"\n", "\\n"}, + {"\t", "\\t"}, + {"\f", "\\f"}, + {"\r", "\\r"} + }; + + /** + * Reverse of {@link #JAVA_CTRL_CHARS_ESCAPE()} for unescaping purposes. + * @return the mapping table + */ + public static String[][] JAVA_CTRL_CHARS_UNESCAPE() { return JAVA_CTRL_CHARS_UNESCAPE.clone(); } + private static final String[][] JAVA_CTRL_CHARS_UNESCAPE = invert(JAVA_CTRL_CHARS_ESCAPE); + + /** + * Used to invert an escape array into an unescape array + * @param array String[][] to be inverted + * @return String[][] inverted array + */ + public static String[][] invert(final String[][] array) { + final String[][] newarray = new String[array.length][2]; + for(int i = 0; i<array.length; i++) { + newarray[i][0] = array[i][1]; + newarray[i][1] = array[i][0]; + } + return newarray; + } + +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/JavaUnicodeEscaper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/JavaUnicodeEscaper.java b/src/main/java/org/apache/commons/text/translate/JavaUnicodeEscaper.java new file mode 100644 index 0000000..a9a186e --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/JavaUnicodeEscaper.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +/** + * Translates codepoints to their Unicode escaped value suitable for Java source. + * + * @since 3.2 + */ +public class JavaUnicodeEscaper extends UnicodeEscaper { + + /** + * <p> + * Constructs a <code>JavaUnicodeEscaper</code> above the specified value (exclusive). + * </p> + * + * @param codepoint + * above which to escape + * @return the newly created {@code UnicodeEscaper} instance + */ + public static org.apache.commons.text.translate.JavaUnicodeEscaper above(final int codepoint) { + return outsideOf(0, codepoint); + } + + /** + * <p> + * Constructs a <code>JavaUnicodeEscaper</code> below the specified value (exclusive). + * </p> + * + * @param codepoint + * below which to escape + * @return the newly created {@code UnicodeEscaper} instance + */ + public static org.apache.commons.text.translate.JavaUnicodeEscaper below(final int codepoint) { + return outsideOf(codepoint, Integer.MAX_VALUE); + } + + /** + * <p> + * Constructs a <code>JavaUnicodeEscaper</code> between the specified values (inclusive). + * </p> + * + * @param codepointLow + * above which to escape + * @param codepointHigh + * below which to escape + * @return the newly created {@code UnicodeEscaper} instance + */ + public static org.apache.commons.text.translate.JavaUnicodeEscaper between(final int codepointLow, final int codepointHigh) { + return new org.apache.commons.text.translate.JavaUnicodeEscaper(codepointLow, codepointHigh, true); + } + + /** + * <p> + * Constructs a <code>JavaUnicodeEscaper</code> outside of the specified values (exclusive). + * </p> + * + * @param codepointLow + * below which to escape + * @param codepointHigh + * above which to escape + * @return the newly created {@code UnicodeEscaper} instance + */ + public static org.apache.commons.text.translate.JavaUnicodeEscaper outsideOf(final int codepointLow, final int codepointHigh) { + return new org.apache.commons.text.translate.JavaUnicodeEscaper(codepointLow, codepointHigh, false); + } + + /** + * <p> + * Constructs a <code>JavaUnicodeEscaper</code> for the specified range. This is the underlying method for the + * other constructors/builders. The <code>below</code> and <code>above</code> boundaries are inclusive when + * <code>between</code> is <code>true</code> and exclusive when it is <code>false</code>. + * </p> + * + * @param below + * int value representing the lowest codepoint boundary + * @param above + * int value representing the highest codepoint boundary + * @param between + * whether to escape between the boundaries or outside them + */ + public JavaUnicodeEscaper(final int below, final int above, final boolean between) { + super(below, above, between); + } + + /** + * Converts the given codepoint to a hex string of the form {@code "\\uXXXX\\uXXXX"} + * + * @param codepoint + * a Unicode code point + * @return the hex string for the given codepoint + */ + @Override + protected String toUtf16Escape(final int codepoint) { + final char[] surrogatePair = Character.toChars(codepoint); + return "\\u" + hex(surrogatePair[0]) + "\\u" + hex(surrogatePair[1]); + } + +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/LookupTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/LookupTranslator.java b/src/main/java/org/apache/commons/text/translate/LookupTranslator.java new file mode 100644 index 0000000..614c86e --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/LookupTranslator.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import java.io.IOException; +import java.io.Writer; +import java.util.HashMap; +import java.util.HashSet; + +/** + * Translates a value using a lookup table. + * + * @since 3.0 + */ +public class LookupTranslator extends CharSequenceTranslator { + + private final HashMap<String, String> lookupMap; + private final HashSet<Character> prefixSet; + private final int shortest; + private final int longest; + + /** + * Define the lookup table to be used in translation + * + * Note that, as of Lang 3.1, the key to the lookup table is converted to a + * java.lang.String. This is because we need the key to support hashCode and + * equals(Object), allowing it to be the key for a HashMap. See LANG-882. + * + * @param lookup CharSequence[][] table of size [*][2] + */ + public LookupTranslator(final CharSequence[]... lookup) { + lookupMap = new HashMap<String, String>(); + prefixSet = new HashSet<Character>(); + int _shortest = Integer.MAX_VALUE; + int _longest = 0; + if (lookup != null) { + for (final CharSequence[] seq : lookup) { + this.lookupMap.put(seq[0].toString(), seq[1].toString()); + this.prefixSet.add(seq[0].charAt(0)); + final int sz = seq[0].length(); + if (sz < _shortest) { + _shortest = sz; + } + if (sz > _longest) { + _longest = sz; + } + } + } + shortest = _shortest; + longest = _longest; + } + + /** + * {@inheritDoc} + */ + @Override + public int translate(final CharSequence input, final int index, final Writer out) throws IOException { + // check if translation exists for the input at position index + if (prefixSet.contains(input.charAt(index))) { + int max = longest; + if (index + longest > input.length()) { + max = input.length() - index; + } + // implement greedy algorithm by trying maximum match first + for (int i = max; i >= shortest; i--) { + final CharSequence subSeq = input.subSequence(index, index + i); + final String result = lookupMap.get(subSeq.toString()); + + if (result != null) { + out.write(result); + return i; + } + } + } + return 0; + } +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/NumericEntityEscaper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/NumericEntityEscaper.java b/src/main/java/org/apache/commons/text/translate/NumericEntityEscaper.java new file mode 100644 index 0000000..a3bf24b --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/NumericEntityEscaper.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import java.io.IOException; +import java.io.Writer; + +/** + * Translates codepoints to their XML numeric entity escaped value. + * + * @since 3.0 + */ +public class NumericEntityEscaper extends CodePointTranslator { + + private final int below; + private final int above; + private final boolean between; + + /** + * <p>Constructs a <code>NumericEntityEscaper</code> for the specified range. This is + * the underlying method for the other constructors/builders. The <code>below</code> + * and <code>above</code> boundaries are inclusive when <code>between</code> is + * <code>true</code> and exclusive when it is <code>false</code>. </p> + * + * @param below int value representing the lowest codepoint boundary + * @param above int value representing the highest codepoint boundary + * @param between whether to escape between the boundaries or outside them + */ + private NumericEntityEscaper(final int below, final int above, final boolean between) { + this.below = below; + this.above = above; + this.between = between; + } + + /** + * <p>Constructs a <code>NumericEntityEscaper</code> for all characters. </p> + */ + public NumericEntityEscaper() { + this(0, Integer.MAX_VALUE, true); + } + + /** + * <p>Constructs a <code>NumericEntityEscaper</code> below the specified value (exclusive). </p> + * + * @param codepoint below which to escape + * @return the newly created {@code NumericEntityEscaper} instance + */ + public static org.apache.commons.text.translate.NumericEntityEscaper below(final int codepoint) { + return outsideOf(codepoint, Integer.MAX_VALUE); + } + + /** + * <p>Constructs a <code>NumericEntityEscaper</code> above the specified value (exclusive). </p> + * + * @param codepoint above which to escape + * @return the newly created {@code NumericEntityEscaper} instance + */ + public static org.apache.commons.text.translate.NumericEntityEscaper above(final int codepoint) { + return outsideOf(0, codepoint); + } + + /** + * <p>Constructs a <code>NumericEntityEscaper</code> between the specified values (inclusive). </p> + * + * @param codepointLow above which to escape + * @param codepointHigh below which to escape + * @return the newly created {@code NumericEntityEscaper} instance + */ + public static org.apache.commons.text.translate.NumericEntityEscaper between(final int codepointLow, final int codepointHigh) { + return new org.apache.commons.text.translate.NumericEntityEscaper(codepointLow, codepointHigh, true); + } + + /** + * <p>Constructs a <code>NumericEntityEscaper</code> outside of the specified values (exclusive). </p> + * + * @param codepointLow below which to escape + * @param codepointHigh above which to escape + * @return the newly created {@code NumericEntityEscaper} instance + */ + public static org.apache.commons.text.translate.NumericEntityEscaper outsideOf(final int codepointLow, final int codepointHigh) { + return new org.apache.commons.text.translate.NumericEntityEscaper(codepointLow, codepointHigh, false); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean translate(final int codepoint, final Writer out) throws IOException { + if(between) { + if (codepoint < below || codepoint > above) { + return false; + } + } else { + if (codepoint >= below && codepoint <= above) { + return false; + } + } + + out.write("&#"); + out.write(Integer.toString(codepoint, 10)); + out.write(';'); + return true; + } +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/NumericEntityUnescaper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/NumericEntityUnescaper.java b/src/main/java/org/apache/commons/text/translate/NumericEntityUnescaper.java new file mode 100644 index 0000000..cd4605d --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/NumericEntityUnescaper.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import java.io.IOException; +import java.io.Writer; +import java.util.Arrays; +import java.util.EnumSet; + +/** + * Translate XML numeric entities of the form &#[xX]?\d+;? to + * the specific codepoint. + * + * Note that the semi-colon is optional. + * + * @since 3.0 + */ +public class NumericEntityUnescaper extends CharSequenceTranslator { + + public static enum OPTION { semiColonRequired, semiColonOptional, errorIfNoSemiColon } + + // TODO?: Create an OptionsSet class to hide some of the conditional logic below + private final EnumSet<OPTION> options; + + /** + * Create a UnicodeUnescaper. + * + * The constructor takes a list of options, only one type of which is currently + * available (whether to allow, error or ignore the semi-colon on the end of a + * numeric entity to being missing). + * + * For example, to support numeric entities without a ';': + * new NumericEntityUnescaper(NumericEntityUnescaper.OPTION.semiColonOptional) + * and to throw an IllegalArgumentException when they're missing: + * new NumericEntityUnescaper(NumericEntityUnescaper.OPTION.errorIfNoSemiColon) + * + * Note that the default behaviour is to ignore them. + * + * @param options to apply to this unescaper + */ + public NumericEntityUnescaper(final OPTION... options) { + if(options.length > 0) { + this.options = EnumSet.copyOf(Arrays.asList(options)); + } else { + this.options = EnumSet.copyOf(Arrays.asList(new OPTION[] { OPTION.semiColonRequired })); + } + } + + /** + * Whether the passed in option is currently set. + * + * @param option to check state of + * @return whether the option is set + */ + public boolean isSet(final OPTION option) { + return options == null ? false : options.contains(option); + } + + /** + * {@inheritDoc} + */ + @Override + public int translate(final CharSequence input, final int index, final Writer out) throws IOException { + final int seqEnd = input.length(); + // Uses -2 to ensure there is something after the &# + if(input.charAt(index) == '&' && index < seqEnd - 2 && input.charAt(index + 1) == '#') { + int start = index + 2; + boolean isHex = false; + + final char firstChar = input.charAt(start); + if(firstChar == 'x' || firstChar == 'X') { + start++; + isHex = true; + + // Check there's more than just an x after the &# + if(start == seqEnd) { + return 0; + } + } + + int end = start; + // Note that this supports character codes without a ; on the end + while(end < seqEnd && ( input.charAt(end) >= '0' && input.charAt(end) <= '9' || + input.charAt(end) >= 'a' && input.charAt(end) <= 'f' || + input.charAt(end) >= 'A' && input.charAt(end) <= 'F' ) ) + { + end++; + } + + final boolean semiNext = end != seqEnd && input.charAt(end) == ';'; + + if(!semiNext) { + if(isSet(OPTION.semiColonRequired)) { + return 0; + } else + if(isSet(OPTION.errorIfNoSemiColon)) { + throw new IllegalArgumentException("Semi-colon required at end of numeric entity"); + } + } + + int entityValue; + try { + if(isHex) { + entityValue = Integer.parseInt(input.subSequence(start, end).toString(), 16); + } else { + entityValue = Integer.parseInt(input.subSequence(start, end).toString(), 10); + } + } catch(final NumberFormatException nfe) { + return 0; + } + + if(entityValue > 0xFFFF) { + final char[] chrs = Character.toChars(entityValue); + out.write(chrs[0]); + out.write(chrs[1]); + } else { + out.write(entityValue); + } + + return 2 + end - start + (isHex ? 1 : 0) + (semiNext ? 1 : 0); + } + return 0; + } +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/OctalUnescaper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/OctalUnescaper.java b/src/main/java/org/apache/commons/text/translate/OctalUnescaper.java new file mode 100644 index 0000000..5801348 --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/OctalUnescaper.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import java.io.IOException; +import java.io.Writer; + +/** + * Translate escaped octal Strings back to their octal values. + * + * For example, "\45" should go back to being the specific value (a %). + * + * Note that this currently only supports the viable range of octal for Java; namely + * 1 to 377. This is because parsing Java is the main use case. + * + * @since 3.0 + */ +public class OctalUnescaper extends CharSequenceTranslator { + + /** + * {@inheritDoc} + */ + @Override + public int translate(final CharSequence input, final int index, final Writer out) throws IOException { + final int remaining = input.length() - index - 1; // how many characters left, ignoring the first \ + final StringBuilder builder = new StringBuilder(); + if(input.charAt(index) == '\\' && remaining > 0 && isOctalDigit(input.charAt(index + 1)) ) { + final int next = index + 1; + final int next2 = index + 2; + final int next3 = index + 3; + + // we know this is good as we checked it in the if block above + builder.append(input.charAt(next)); + + if(remaining > 1 && isOctalDigit(input.charAt(next2))) { + builder.append(input.charAt(next2)); + if(remaining > 2 && isZeroToThree(input.charAt(next)) && isOctalDigit(input.charAt(next3))) { + builder.append(input.charAt(next3)); + } + } + + out.write( Integer.parseInt(builder.toString(), 8) ); + return 1 + builder.length(); + } + return 0; + } + + /** + * Checks if the given char is an octal digit. Octal digits are the character representations of the digits 0 to 7. + * @param ch the char to check + * @return true if the given char is the character representation of one of the digits from 0 to 7 + */ + private boolean isOctalDigit(final char ch) { + return ch >= '0' && ch <= '7'; + } + + /** + * Checks if the given char is the character representation of one of the digit from 0 to 3. + * @param ch the char to check + * @return true if the given char is the character representation of one of the digits from 0 to 3 + */ + private boolean isZeroToThree(final char ch) { + return ch >= '0' && ch <= '3'; + } +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/UnicodeEscaper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/UnicodeEscaper.java b/src/main/java/org/apache/commons/text/translate/UnicodeEscaper.java new file mode 100644 index 0000000..398a0e9 --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/UnicodeEscaper.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import java.io.IOException; +import java.io.Writer; + +/** + * Translates codepoints to their Unicode escaped value. + * + * @since 3.0 + */ +public class UnicodeEscaper extends CodePointTranslator { + + private final int below; + private final int above; + private final boolean between; + + /** + * <p>Constructs a <code>UnicodeEscaper</code> for all characters. </p> + */ + public UnicodeEscaper(){ + this(0, Integer.MAX_VALUE, true); + } + + /** + * <p>Constructs a <code>UnicodeEscaper</code> for the specified range. This is + * the underlying method for the other constructors/builders. The <code>below</code> + * and <code>above</code> boundaries are inclusive when <code>between</code> is + * <code>true</code> and exclusive when it is <code>false</code>. </p> + * + * @param below int value representing the lowest codepoint boundary + * @param above int value representing the highest codepoint boundary + * @param between whether to escape between the boundaries or outside them + */ + protected UnicodeEscaper(final int below, final int above, final boolean between) { + this.below = below; + this.above = above; + this.between = between; + } + + /** + * <p>Constructs a <code>UnicodeEscaper</code> below the specified value (exclusive). </p> + * + * @param codepoint below which to escape + * @return the newly created {@code UnicodeEscaper} instance + */ + public static org.apache.commons.text.translate.UnicodeEscaper below(final int codepoint) { + return outsideOf(codepoint, Integer.MAX_VALUE); + } + + /** + * <p>Constructs a <code>UnicodeEscaper</code> above the specified value (exclusive). </p> + * + * @param codepoint above which to escape + * @return the newly created {@code UnicodeEscaper} instance + */ + public static org.apache.commons.text.translate.UnicodeEscaper above(final int codepoint) { + return outsideOf(0, codepoint); + } + + /** + * <p>Constructs a <code>UnicodeEscaper</code> outside of the specified values (exclusive). </p> + * + * @param codepointLow below which to escape + * @param codepointHigh above which to escape + * @return the newly created {@code UnicodeEscaper} instance + */ + public static org.apache.commons.text.translate.UnicodeEscaper outsideOf(final int codepointLow, final int codepointHigh) { + return new org.apache.commons.text.translate.UnicodeEscaper(codepointLow, codepointHigh, false); + } + + /** + * <p>Constructs a <code>UnicodeEscaper</code> between the specified values (inclusive). </p> + * + * @param codepointLow above which to escape + * @param codepointHigh below which to escape + * @return the newly created {@code UnicodeEscaper} instance + */ + public static org.apache.commons.text.translate.UnicodeEscaper between(final int codepointLow, final int codepointHigh) { + return new org.apache.commons.text.translate.UnicodeEscaper(codepointLow, codepointHigh, true); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean translate(final int codepoint, final Writer out) throws IOException { + if (between) { + if (codepoint < below || codepoint > above) { + return false; + } + } else { + if (codepoint >= below && codepoint <= above) { + return false; + } + } + + // TODO: Handle potential + sign per various Unicode escape implementations + if (codepoint > 0xffff) { + out.write(toUtf16Escape(codepoint)); + } else { + out.write("\\u"); + out.write(HEX_DIGITS[(codepoint >> 12) & 15]); + out.write(HEX_DIGITS[(codepoint >> 8) & 15]); + out.write(HEX_DIGITS[(codepoint >> 4) & 15]); + out.write(HEX_DIGITS[(codepoint) & 15]); + } + return true; + } + + /** + * Converts the given codepoint to a hex string of the form {@code "\\uXXXX"} + * + * @param codepoint + * a Unicode code point + * @return the hex string for the given codepoint + * + * @since 3.2 + */ + protected String toUtf16Escape(final int codepoint) { + return "\\u" + hex(codepoint); + } +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/UnicodeUnescaper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/UnicodeUnescaper.java b/src/main/java/org/apache/commons/text/translate/UnicodeUnescaper.java new file mode 100644 index 0000000..b92f421 --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/UnicodeUnescaper.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import java.io.IOException; +import java.io.Writer; + +/** + * Translates escaped Unicode values of the form \\u+\d\d\d\d back to + * Unicode. It supports multiple 'u' characters and will work with or + * without the +. + * + * @since 3.0 + */ +public class UnicodeUnescaper extends CharSequenceTranslator { + + /** + * {@inheritDoc} + */ + @Override + public int translate(final CharSequence input, final int index, final Writer out) throws IOException { + if (input.charAt(index) == '\\' && index + 1 < input.length() && input.charAt(index + 1) == 'u') { + // consume optional additional 'u' chars + int i = 2; + while (index + i < input.length() && input.charAt(index + i) == 'u') { + i++; + } + + if (index + i < input.length() && input.charAt(index + i) == '+') { + i++; + } + + if (index + i + 4 <= input.length()) { + // Get 4 hex digits + final CharSequence unicode = input.subSequence(index + i, index + i + 4); + + try { + final int value = Integer.parseInt(unicode.toString(), 16); + out.write((char) value); + } catch (final NumberFormatException nfe) { + throw new IllegalArgumentException("Unable to parse unicode value: " + unicode, nfe); + } + return i + 4; + } + throw new IllegalArgumentException("Less than 4 hex digits in unicode value: '" + input.subSequence(index, input.length()) + + "' due to end of CharSequence"); + } + return 0; + } +} http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/UnicodeUnpairedSurrogateRemover.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/UnicodeUnpairedSurrogateRemover.java b/src/main/java/org/apache/commons/text/translate/UnicodeUnpairedSurrogateRemover.java new file mode 100644 index 0000000..cf18d62 --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/UnicodeUnpairedSurrogateRemover.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.commons.text.translate; + +import java.io.IOException; +import java.io.Writer; + +/** + * Helper subclass to CharSequenceTranslator to remove unpaired surrogates. + */ +public class UnicodeUnpairedSurrogateRemover extends CodePointTranslator { + /** + * Implementation of translate that throws out unpaired surrogates. + * {@inheritDoc} + */ + @Override + public boolean translate(final int codepoint, final Writer out) throws IOException { + if (codepoint >= Character.MIN_SURROGATE && codepoint <= Character.MAX_SURROGATE) { + // It's a surrogate. Write nothing and say we've translated. + return true; + } + // It's not a surrogate. Don't translate it. + return false; + } +} + http://git-wip-us.apache.org/repos/asf/commons-text/blob/617869b2/src/main/java/org/apache/commons/text/translate/package-info.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/text/translate/package-info.java b/src/main/java/org/apache/commons/text/translate/package-info.java new file mode 100644 index 0000000..722d7c9 --- /dev/null +++ b/src/main/java/org/apache/commons/text/translate/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * <p> An API for creating text translation routines from a set of smaller building blocks. Initially created to make it + * possible for the user to customize the rules in the StringEscapeUtils class.</p> + * <p>These classes are immutable, and therefore thread-safe.</p> + * + * @since 3.0 + */ +package org.apache.commons.text.translate;
