Author: sebb
Date: Thu Mar 22 19:04:41 2012
New Revision: 1303988
URL: http://svn.apache.org/viewvc?rev=1303988&view=rev
Log:
CSV-81 Token.Type.isReady could perhaps be removed
Not removed, but only set on EOF if there is data to return
Modified:
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java
Modified:
commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java
URL:
http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java?rev=1303988&r1=1303987&r2=1303988&view=diff
==============================================================================
--- commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java
(original)
+++ commons/proper/csv/trunk/src/main/java/org/apache/commons/csv/CSVLexer.java
Thu Mar 22 19:04:41 2012
@@ -65,6 +65,7 @@ class CSVLexer extends Lexer {
// reached end of file without any content (empty line at the
end)
if (isEndOfFile(c)) {
tkn.type = EOF;
+ // don't set tkn.isReady here because no content
return tkn;
}
}
@@ -73,11 +74,12 @@ class CSVLexer extends Lexer {
// did we reach eof during the last iteration already ? EOF
if (isEndOfFile(lastChar) || (!isDelimiter(lastChar) &&
isEndOfFile(c))) {
tkn.type = EOF;
+ // don't set tkn.isReady here because no content
return tkn;
}
// important: make sure a new char gets consumed in each iteration
- while (!tkn.isReady && tkn.type != EOF) {
+ while (tkn.type == INVALID) {
// ignore whitespaces at beginning of a token
if (leadingSpacesIgnored) {
while (isWhitespace(c) && !eol) {
@@ -94,12 +96,10 @@ class CSVLexer extends Lexer {
} else if (isDelimiter(c)) {
// empty token return TOKEN("")
tkn.type = TOKEN;
- tkn.isReady = true;
} else if (eol) {
// empty token return EORECORD("")
//noop: tkn.content.append("");
tkn.type = EORECORD;
- tkn.isReady = true;
} else if (isEncapsulator(c)) {
// consume encapsulated token
encapsulatedTokenLexer(tkn, c);
@@ -107,7 +107,7 @@ class CSVLexer extends Lexer {
// end of file return EOF()
//noop: tkn.content.append("");
tkn.type = EOF;
- tkn.isReady = true;
+ tkn.isReady = true; // there is data at EOF
} else {
// next token must be a simple token
// add removed blanks when not ignoring whitespace chars...
@@ -139,17 +139,15 @@ class CSVLexer extends Lexer {
if (isEndOfLine(c)) {
// end of record
tkn.type = EORECORD;
- tkn.isReady = true;
break;
} else if (isEndOfFile(c)) {
// end of file
tkn.type = EOF;
- tkn.isReady = true;
+ tkn.isReady = true; // There is data at EOF
break;
} else if (isDelimiter(c)) {
// end of token
tkn.type = TOKEN;
- tkn.isReady = true;
break;
} else if (isEscape(c)) {
tkn.content.append((char) readEscape(c));
@@ -201,16 +199,14 @@ class CSVLexer extends Lexer {
c = in.read();
if (isDelimiter(c)) {
tkn.type = TOKEN;
- tkn.isReady = true;
return tkn;
} else if (isEndOfFile(c)) {
tkn.type = EOF;
- tkn.isReady = true;
+ tkn.isReady = true; // There is data at EOF
return tkn;
} else if (isEndOfLine(c)) {
// ok eo token reached
tkn.type = EORECORD;
- tkn.isReady = true;
return tkn;
} else if (!isWhitespace(c)) {
// error invalid char between token and next
delimiter
