This is an automated email from the ASF dual-hosted git repository.
onders pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 248cb88ffca1dfe5ae26f4dbbac6348a5b042523
Merge: 081474d 9d946dc
Author: onders <ond...@apache.org>
AuthorDate: Sat Oct 6 00:03:32 2018 +0300
CAMEL-12698: Use the Stream API to read files instead of Scanner
.../camel/dataformat/bindy/WrappedException.java | 38 ++++++++
.../dataformat/bindy/csv/BindyCsvDataFormat.java | 108 +++++++++++----------
.../bindy/kvp/BindyKeyValuePairDataFormat.java | 85 +++++++++-------
...indySimpleCsvUnmarshallUnicodeNextLineTest.java | 72 ++++++++++++++
...BindySimpleKeyValuePairUnicodeNextLineTest.java | 100 +++++++++++++++++++
.../bindy/model/unicode/LocationRecord.java | 51 ++++++++++
...pleCsvUnmarshallUnicodeNextLineTest-context.xml | 34 +++++++
...mpleKeyValuePairUnicodeNextLineTest-context.xml | 34 +++++++
8 files changed, 437 insertions(+), 85 deletions(-)
diff --cc
components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/csv/BindyCsvDataFormat.java
index 82edf8e,5906982..4f1c927
---
a/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/csv/BindyCsvDataFormat.java
+++
b/components/camel-bindy/src/main/java/org/apache/camel/dataformat/bindy/csv/BindyCsvDataFormat.java
@@@ -147,41 -148,59 +149,60 @@@ public class BindyCsvDataFormat extend
if (checkEmptyStream(factory, inputStream)) {
return models;
}
-
+
in = new InputStreamReader(inputStream,
IOHelper.getCharsetName(exchange));
- // Scanner is used to read big file
- scanner = new Scanner(in);
-
// Retrieve the separator defined to split the record
String separator = factory.getSeparator();
String quote = factory.getQuote();
- Boolean removeQuotes = factory.getRemoveQuotes();
ObjectHelper.notNull(separator, "The separator has not been
defined in the annotation @CsvRecord or not instantiated during initModel.");
+
- int count = 0;
+ AtomicInteger count = new AtomicInteger(0);
- // If the first line of the CSV file contains columns name, then
we
- // skip this line
- if (factory.getSkipFirstLine()) {
- // Check if scanner is empty
- if (scanner.hasNextLine()) {
- scanner.nextLine();
- }
- }
+ // Use a Stream to stream a file across.
+ try (Stream<String> lines = new BufferedReader(in).lines()) {
+ int linesToSkip = 0;
- while (scanner.hasNextLine()) {
+ // If the first line of the CSV file contains columns name,
then we
+ // skip this line
+ if (factory.getSkipFirstLine()) {
+ linesToSkip = 1;
+ }
- // Read the line
- String line = scanner.nextLine().trim();
+ // Consume the lines in the file via a consumer method,
passing in state as necessary.
+ // If the internals of the consumer fail, we unrap the
checked exception upstream.
+ try {
+ lines.skip(linesToSkip)
+ .forEachOrdered(consumeFile(factory, models,
separator, quote, count));
+ } catch (WrappedException e) {
+ throw e.getWrappedException();
+ }
- if (ObjectHelper.isEmpty(line)) {
- // skip if line is empty
- continue;
+ // BigIntegerFormatFactory if models list is empty or not
+ // If this is the case (correspond to an empty stream, ...)
+ if (models.size() == 0) {
+ throw new java.lang.IllegalArgumentException("No records
have been defined in the CSV");
+ } else {
+ return extractUnmarshalResult(models);
}
+ }
+ } finally {
+ if (in != null) {
+ IOHelper.close(in, "in", LOG);
+ }
+ }
+
+ }
+ private Consumer<String> consumeFile(BindyCsvFactory factory,
List<Map<String, Object>> models,
+ String separator, String quote,
AtomicInteger count) {
+ return line -> {
+ try {
+ // Trim the line coming in to remove any trailing whitespace
+ String trimmedLine = line.trim();
// Increment counter
- count++;
+ count.incrementAndGet();
+ Map<String, Object> model;
// Create POJO where CSV data will be stored
model = factory.factory();
@@@ -201,22 -220,21 +222,24 @@@
separators.add(separators.get(separators.size() - 1));
}
- String[] tokens = pattern.split(line,
factory.getAutospanLine() ? factory.getMaxpos() : -1);
+ String[] tokens = pattern.split(trimmedLine,
factory.getAutospanLine() ? factory.getMaxpos() : -1);
++
List<String> result = Arrays.asList(tokens);
+
// must unquote tokens before use
- if (removeQuotes) {
- result = unquoteTokens(result, separators, quote);
- }
+
- if (result.size() == 0 || result.isEmpty()) {
- throw new java.lang.IllegalArgumentException("No records
have been defined in the CSV");
+ result = unquoteTokens(result, separators, quote);
+
+ if (result.isEmpty()) {
+ throw new IllegalArgumentException("No records have been
defined in the CSV");
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Size of the record splitted : {}",
result.size());
}
// Bind data from CSV record with model classes
- factory.bind(getCamelContext(), result, model, count);
++
+ factory.bind(getCamelContext(), result, model,
count.get());
// Link objects together
factory.link(model);
