This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
The following commit(s) were added to refs/heads/master by this push:
new 030fb8e [CSV-239] Add CSVRecord.getHeaderNames and allow duplicate
headers (#41)
030fb8e is described below
commit 030fb8e37c4024b24fac2b5404300449a6741699
Author: Dave Moten <davidmo...@gmail.com>
AuthorDate: Fri May 24 22:11:17 2019 +1000
[CSV-239] Add CSVRecord.getHeaderNames and allow duplicate headers (#41)
* [CSV-239] Cannot get headers in column order from CSVRecord.
* getHeaderNames returns all headers in column order including repeats
which are allowed as per RFC 4180
* add CSVFormat.withAllowDuplicateHeaderNames()
* [CSV-239] Cannot get headers in column order from CSVRecord.
* only wrap headerNames with unmodifiableList if non-empty
* fix and enhance CSVRecord.toMap javadoc
* [CSV-239] Cannot get headers in column order from CSVRecord.
* fix exception messages
* [CSV-239] Cannot get headers in column order from CSVRecord.
* fix whitespace
* [CSV-239] Cannot get headers in column order from CSVRecord.
* simplify if statement
* [CSV-239] Cannot get headers in column order from CSVRecord.
* fix indentation
* add javadoc to Headers class
* rename method to createHeaders
* use String.format to build error message
* initialize header names List with appropriate size
---
.../java/org/apache/commons/csv/CSVFormat.java | 77 ++++++++++++++++------
.../java/org/apache/commons/csv/CSVParser.java | 63 +++++++++++++-----
.../java/org/apache/commons/csv/CSVRecord.java | 2 +-
.../java/org/apache/commons/csv/CSVParserTest.java | 25 ++++++-
4 files changed, 128 insertions(+), 39 deletions(-)
diff --git a/src/main/java/org/apache/commons/csv/CSVFormat.java
b/src/main/java/org/apache/commons/csv/CSVFormat.java
index 6f77bef..33130b0 100644
--- a/src/main/java/org/apache/commons/csv/CSVFormat.java
+++ b/src/main/java/org/apache/commons/csv/CSVFormat.java
@@ -260,12 +260,13 @@ public final class CSVFormat implements Serializable {
* <li>{@code withQuote('"')}</li>
* <li>{@code withRecordSeparator("\r\n")}</li>
* <li>{@code withIgnoreEmptyLines(true)}</li>
+ * <li>{@code withAllowDuplicateHeaderNames(true)}</li>
* </ul>
*
* @see Predefined#Default
*/
public static final CSVFormat DEFAULT = new CSVFormat(COMMA,
DOUBLE_QUOTE_CHAR, null, null, null, false, true, CRLF,
- null, null, null, false, false, false, false, false, false);
+ null, null, null, false, false, false, false, false, false, true);
/**
* Excel file format (using a comma as the value delimiter). Note that the
actual value delimiter used by Excel is
@@ -288,6 +289,7 @@ public final class CSVFormat implements Serializable {
* <li>{@code {@link #withRecordSeparator(String)
withRecordSeparator("\r\n")}}</li>
* <li>{@code {@link #withIgnoreEmptyLines(boolean)
withIgnoreEmptyLines(false)}}</li>
* <li>{@code {@link #withAllowMissingColumnNames(boolean)
withAllowMissingColumnNames(true)}}</li>
+ * <li>{@code {@link #withAllowDuplicateHeaderNames(boolean)
withAllowDuplicateHeaderNames(true)}}</li>
* </ul>
* <p>
* Note: This is currently like {@link #RFC4180} plus {@link
#withAllowMissingColumnNames(boolean)
@@ -671,7 +673,7 @@ public final class CSVFormat implements Serializable {
*/
public static CSVFormat newFormat(final char delimiter) {
return new CSVFormat(delimiter, null, null, null, null, false, false,
null, null, null, null, false, false,
- false, false, false, false);
+ false, false, false, false, true);
}
/**
@@ -721,6 +723,8 @@ public final class CSVFormat implements Serializable {
private final boolean trim;
private final boolean autoFlush;
+
+ private final boolean allowDuplicateHeaderNames;
/**
* Creates a customized CSV format.
@@ -766,7 +770,7 @@ public final class CSVFormat implements Serializable {
final boolean ignoreEmptyLines, final String recordSeparator,
final String nullString,
final Object[] headerComments, final String[] header, final
boolean skipHeaderRecord,
final boolean allowMissingColumnNames, final boolean
ignoreHeaderCase, final boolean trim,
- final boolean trailingDelimiter, final boolean autoFlush) {
+ final boolean trailingDelimiter, final boolean autoFlush, final
boolean allowDuplicateHeaderNames) {
this.delimiter = delimiter;
this.quoteCharacter = quoteChar;
this.quoteMode = quoteMode;
@@ -785,6 +789,7 @@ public final class CSVFormat implements Serializable {
this.trim = trim;
this.autoFlush = autoFlush;
this.quotedNullString = quoteCharacter + nullString + quoteCharacter;
+ this.allowDuplicateHeaderNames = allowDuplicateHeaderNames;
validate();
}
@@ -1686,7 +1691,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withAllowMissingColumnNames(final boolean
allowMissingColumnNames) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -1701,7 +1707,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withAutoFlush(final boolean autoFlush) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -1736,7 +1743,8 @@ public final class CSVFormat implements Serializable {
}
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -1754,7 +1762,8 @@ public final class CSVFormat implements Serializable {
}
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -1785,7 +1794,8 @@ public final class CSVFormat implements Serializable {
}
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escape, ignoreSurroundingSpaces,
ignoreEmptyLines, recordSeparator, nullString, headerComments,
header, skipHeaderRecord,
- allowMissingColumnNames, ignoreHeaderCase, trim,
trailingDelimiter, autoFlush);
+ allowMissingColumnNames, ignoreHeaderCase, trim,
trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -1941,7 +1951,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withHeader(final String... header) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -1962,7 +1973,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withHeaderComments(final Object... headerComments) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -1987,7 +1999,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withIgnoreEmptyLines(final boolean ignoreEmptyLines) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -2013,7 +2026,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withIgnoreHeaderCase(final boolean ignoreHeaderCase) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -2038,7 +2052,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withIgnoreSurroundingSpaces(final boolean
ignoreSurroundingSpaces) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -2057,7 +2072,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withNullString(final String nullString) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -2088,7 +2104,8 @@ public final class CSVFormat implements Serializable {
}
return new CSVFormat(delimiter, quoteChar, quoteMode, commentMarker,
escapeCharacter, ignoreSurroundingSpaces,
ignoreEmptyLines, recordSeparator, nullString, headerComments,
header, skipHeaderRecord,
- allowMissingColumnNames, ignoreHeaderCase, trim,
trailingDelimiter, autoFlush);
+ allowMissingColumnNames, ignoreHeaderCase, trim,
trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -2102,7 +2119,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withQuoteMode(final QuoteMode quoteModePolicy) {
return new CSVFormat(delimiter, quoteCharacter, quoteModePolicy,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -2140,7 +2158,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withRecordSeparator(final String recordSeparator) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -2167,7 +2186,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withSkipHeaderRecord(final boolean skipHeaderRecord) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -2208,7 +2228,8 @@ public final class CSVFormat implements Serializable {
public CSVFormat withTrailingDelimiter(final boolean trailingDelimiter) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
/**
@@ -2233,6 +2254,22 @@ public final class CSVFormat implements Serializable {
public CSVFormat withTrim(final boolean trim) {
return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
- skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush);
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
}
+
+ public CSVFormat withAllowDuplicateHeaderNames(boolean
allowDuplicateHeaderNames) {
+ return new CSVFormat(delimiter, quoteCharacter, quoteMode,
commentMarker, escapeCharacter,
+ ignoreSurroundingSpaces, ignoreEmptyLines, recordSeparator,
nullString, headerComments, header,
+ skipHeaderRecord, allowMissingColumnNames, ignoreHeaderCase,
trim, trailingDelimiter, autoFlush,
+ allowDuplicateHeaderNames);
+ }
+
+ public CSVFormat withAllowDuplicateHeaderNames() {
+ return withAllowDuplicateHeaderNames(true);
+ }
+
+ public boolean getAllowDuplicateHeaderNames() {
+ return allowDuplicateHeaderNames;
+ }
}
diff --git a/src/main/java/org/apache/commons/csv/CSVParser.java
b/src/main/java/org/apache/commons/csv/CSVParser.java
index 15357c0..eff7402 100644
--- a/src/main/java/org/apache/commons/csv/CSVParser.java
+++ b/src/main/java/org/apache/commons/csv/CSVParser.java
@@ -410,8 +410,9 @@ public final class CSVParser implements
Iterable<CSVRecord>, Closeable {
this.format = format;
this.lexer = new Lexer(format, new ExtendedBufferedReader(reader));
this.csvRecordIterator = new CSVRecordIterator();
- this.headerMap = createHeaderMap(); // 1st
- this.headerNames = createHeaderNames(this.headerMap); // 2nd
+ Headers headers = createHeaders();
+ this.headerMap = headers.headerMap;
+ this.headerNames = headers.headerNames;
this.characterOffset = characterOffset;
this.recordNumber = recordNumber - 1;
}
@@ -446,13 +447,34 @@ public final class CSVParser implements
Iterable<CSVRecord>, Closeable {
}
/**
+ * Header information based on name and position.
+ */
+ private static final class Headers {
+ /**
+ * Header column positions (0-based)
+ */
+ final Map<String, Integer> headerMap;
+
+ /**
+ * Header names in column order
+ */
+ final List<String> headerNames;
+
+ Headers(Map<String, Integer> headerMap, List<String> headerNames) {
+ this.headerMap = headerMap;
+ this.headerNames = headerNames;
+ }
+ }
+
+ /**
* Creates the name to index mapping if the format defines a header.
*
* @return null if the format has no header.
* @throws IOException if there is a problem reading the header or
skipping the first record
*/
- private Map<String, Integer> createHeaderMap() throws IOException {
+ private Headers createHeaders() throws IOException {
Map<String, Integer> hdrMap = null;
+ List<String> headerNames = null;
final String[] formatHeader = this.format.getHeader();
if (formatHeader != null) {
hdrMap = createEmptyHeaderMap();
@@ -476,27 +498,34 @@ public final class CSVParser implements
Iterable<CSVRecord>, Closeable {
final String header = headerRecord[i];
final boolean containsHeader = header == null ? false :
hdrMap.containsKey(header);
final boolean emptyHeader = header == null ||
header.trim().isEmpty();
- if (containsHeader && (!emptyHeader ||
!this.format.getAllowMissingColumnNames())) {
- throw new IllegalArgumentException("The header
contains a duplicate name: \"" + header
- + "\" in " + Arrays.toString(headerRecord));
+ if (containsHeader) {
+ if (!emptyHeader &&
!this.format.getAllowDuplicateHeaderNames()) {
+ throw new IllegalArgumentException(
+ String.format("The header contains a
duplicate name: \"%s\" in %s."
+ + " If this is valid then use
CSVFormat.withAllowDuplicateHeaderNames().",
+ header,
Arrays.toString(headerRecord)));
+ }
+ if (emptyHeader &&
!this.format.getAllowMissingColumnNames()) {
+ throw new IllegalArgumentException(
+ "A header name is missing in " +
Arrays.toString(headerRecord));
+ }
}
if (header != null) {
hdrMap.put(header, Integer.valueOf(i));
+ if (headerNames == null) {
+ headerNames = new
ArrayList<>(headerRecord.length);
+ }
+ headerNames.add(header);
}
}
}
+ }
+ if (headerNames == null) {
+ headerNames = Collections.emptyList(); //immutable
+ } else {
+ headerNames = Collections.unmodifiableList(headerNames);
}
- return hdrMap;
- }
-
- private List<String> createHeaderNames(final Map<String, Integer>
headerMap) {
- // @formatter:off
- return headerMap == null ? null
- : headerMap.entrySet().stream()
- .sorted(Map.Entry.comparingByValue())
- .map(Map.Entry::getKey)
- .collect(Collectors.collectingAndThen(Collectors.toList(),
Collections::unmodifiableList));
- // @formatter:on
+ return new Headers(hdrMap, headerNames);
}
/**
diff --git a/src/main/java/org/apache/commons/csv/CSVRecord.java
b/src/main/java/org/apache/commons/csv/CSVRecord.java
index acc4c8d..99dce26 100644
--- a/src/main/java/org/apache/commons/csv/CSVRecord.java
+++ b/src/main/java/org/apache/commons/csv/CSVRecord.java
@@ -265,7 +265,7 @@ public final class CSVRecord implements Serializable,
Iterable<String> {
}
/**
- * Copies this record into a new Map. The new map is not connect
+ * Copies this record into a new Map of header name to record value.
*
* @return A new Map. The map is empty if the record has no headers.
*/
diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java
b/src/test/java/org/apache/commons/csv/CSVParserTest.java
index 57bed48..f71e479 100644
--- a/src/test/java/org/apache/commons/csv/CSVParserTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java
@@ -43,6 +43,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -292,11 +293,25 @@ public class CSVParserTest {
}
@Test(expected = IllegalArgumentException.class)
- public void testDuplicateHeaders() throws Exception {
+ public void testDuplicateHeadersNotAllowed() throws Exception {
+ CSVParser.parse("a,b,a\n1,2,3\nx,y,z",
+ CSVFormat.DEFAULT.withHeader(new String[]
{}).withAllowDuplicateHeaderNames(false));
+ }
+
+ @Test
+ public void testDuplicateHeadersAllowedByDefault() throws Exception {
CSVParser.parse("a,b,a\n1,2,3\nx,y,z",
CSVFormat.DEFAULT.withHeader(new String[] {}));
}
@Test
+ public void testEmptyFileHeaderParsing() throws Exception {
+ try (final CSVParser parser = CSVParser.parse("",
CSVFormat.DEFAULT.withFirstRecordAsHeader())) {
+ assertNull(parser.nextRecord());
+ assertTrue(parser.getHeaderNames().isEmpty());
+ }
+ }
+
+ @Test
public void testEmptyFile() throws Exception {
try (final CSVParser parser = CSVParser.parse("", CSVFormat.DEFAULT)) {
assertNull(parser.nextRecord());
@@ -1151,6 +1166,14 @@ public class CSVParserTest {
assertEquals("3", record.get("Z"));
Assert.assertEquals(3, record.size());
}
+
+ @Test
+ public void testRepeatedHeadersAreReturnedInCSVRecordHeaderNames() throws
IOException {
+ final Reader in = new
StringReader("header1,header2,header1\n1,2,3\n4,5,6");
+ final Iterator<CSVRecord> records =
CSVFormat.DEFAULT.withFirstRecordAsHeader().withTrim().parse(in).iterator();
+ final CSVRecord record = records.next();
+ assertEquals(Arrays.asList("header1", "header2", "header1"),
record.getParser().getHeaderNames());
+ }
private void validateLineNumbers(final String lineSeparator) throws
IOException {
try (final CSVParser parser = CSVParser.parse("a" + lineSeparator +
"b" + lineSeparator + "c",
