Author: sebb
Date: Thu Mar 29 15:01:04 2012
New Revision: 1306890
URL: http://svn.apache.org/viewvc?rev=1306890&view=rev
Log:
Improve escape tests
Fix bug in readEscape() - was not handling EOF
Modified:
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java
commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java
Modified:
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java
URL:
http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java?rev=1306890&r1=1306889&r2=1306890&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java
(original)
+++ commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/Lexer.java
Thu Mar 29 15:01:04 2012
@@ -60,6 +60,7 @@ abstract class Lexer {
return in.getLineNumber();
}
+ // TODO escape handling needs more work
int readEscape() throws IOException {
// assume c is the escape char (normally a backslash)
int c = in.read();
@@ -74,6 +75,8 @@ abstract class Lexer {
return '\b';
case 'f':
return '\f';
+ case ExtendedBufferedReader.END_OF_STREAM:
+ throw new IOException("EOF whilst processing escape sequence");
default:
return c;
}
Modified:
commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java
URL:
http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java?rev=1306890&r1=1306889&r2=1306890&view=diff
==============================================================================
---
commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java
(original)
+++
commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/CSVLexerTest.java
Thu Mar 29 15:01:04 2012
@@ -147,28 +147,66 @@ public class CSVLexerTest {
}
- // simple token with escaping
+ // simple token with escaping not enabled
@Test
public void testNextToken3() throws IOException {
/* file: a,\,,b
* \,,
*/
- String code = "a,\\,,b\n\\,,";
- CSVFormat format = CSVFormat.DEFAULT.withCommentStart('#');
+ String code = "a,\\,,b\\\n\\,,";
+ CSVFormat format = CSVFormat.DEFAULT;
+ assertFalse(format.isEscaping());
Lexer parser = getLexer(code, format);
assertTokenEquals(TOKEN, "a", parser.nextToken(new Token()));
// an unquoted single backslash is not an escape char
assertTokenEquals(TOKEN, "\\", parser.nextToken(new Token()));
assertTokenEquals(TOKEN, "", parser.nextToken(new Token()));
- assertTokenEquals(EORECORD, "b", parser.nextToken(new Token()));
+ assertTokenEquals(EORECORD, "b\\", parser.nextToken(new Token()));
// an unquoted single backslash is not an escape char
assertTokenEquals(TOKEN, "\\", parser.nextToken(new Token()));
assertTokenEquals(TOKEN, "", parser.nextToken(new Token()));
assertTokenEquals(EOF, "", parser.nextToken(new Token()));
}
- // encapsulator tokenizer (sinle line)
+ // simple token with escaping enabled
+ @Test
+ public void testNextToken3Escaping() throws IOException {
+ /* file: a,\,,b
+ * \,,
+ */
+ String code = "a,\\,,b\\\\\n\\,,\\\nc,d\\\n";
+ CSVFormat format = CSVFormat.DEFAULT.withEscape('\\');
+ assertTrue(format.isEscaping());
+ Lexer parser = getLexer(code, format);
+
+ assertTokenEquals(TOKEN, "a", parser.nextToken(new Token()));
+ assertTokenEquals(TOKEN, ",", parser.nextToken(new Token()));
+ assertTokenEquals(EORECORD, "b\\", parser.nextToken(new Token()));
+ assertTokenEquals(TOKEN, ",", parser.nextToken(new Token()));
+ assertTokenEquals(TOKEN, "\nc", parser.nextToken(new Token()));
+ assertTokenEquals(EOF, "d\n", parser.nextToken(new Token()));
+ assertTokenEquals(EOF, "", parser.nextToken(new Token()));
+ }
+
+ // simple token with escaping enabled
+ @Test
+ public void testNextToken3BadEscaping() throws IOException {
+ String code = "a,b,c\\";
+ CSVFormat format = CSVFormat.DEFAULT.withEscape('\\');
+ assertTrue(format.isEscaping());
+ Lexer parser = getLexer(code, format);
+
+ assertTokenEquals(TOKEN, "a", parser.nextToken(new Token()));
+ assertTokenEquals(TOKEN, "b", parser.nextToken(new Token()));
+ try {
+ Token tkn = parser.nextToken(new Token());
+ fail("Expected IOE, found "+tkn);
+ } catch (IOException e) {
+ }
+ }
+
+ // encapsulator tokenizer (single line)
@Test
public void testNextToken4() throws IOException {
/* file: a,"foo",b
