This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 36e26cd8471 camel-parquet-avro: Add the capability to lazy load
records during unmarshal (#12385)
36e26cd8471 is described below
commit 36e26cd84712ed971b6443aeeb7943365da2ccdd
Author: Derek Marsh <dmars...@users.noreply.github.com>
AuthorDate: Sun Dec 10 11:43:24 2023 -0600
camel-parquet-avro: Add the capability to lazy load records during
unmarshal (#12385)
* lazy load parquet records to avoid memory issues on large files.
* include generated files from addition of lazy load property to
parquet-avro.
---
.../camel/catalog/dataformats/parquetAvro.json | 3 +-
.../apache/camel/catalog/models/parquetAvro.json | 3 +-
.../apache/camel/catalog/schemas/camel-spring.xsd | 9 ++
.../avro/ParquetAvroDataFormatConfigurer.java | 2 +
.../camel/dataformat/parquet/avro/parquetAvro.json | 3 +-
.../parquet/avro/ParquetAvroDataFormat.java | 45 +++++++--
.../dataformat/parquet/avro/ParquetIterator.java | 68 ++++++++++++++
.../parquet/avro/ParquetUnmarshalOnCompletion.java | 41 ++++++++
...ParquetAvroDataFormatUnmarshalLazyLoadTest.java | 103 +++++++++++++++++++++
.../apache/camel/model/dataformat/parquetAvro.json | 3 +-
.../model/dataformat/ParquetAvroDataFormat.java | 38 ++++++++
.../java/org/apache/camel/xml/in/ModelParser.java | 1 +
.../java/org/apache/camel/xml/out/ModelWriter.java | 1 +
.../org/apache/camel/yaml/out/ModelWriter.java | 1 +
.../dsl/yaml/deserializers/ModelDeserializers.java | 6 ++
.../generated/resources/schema/camelYamlDsl.json | 5 +
16 files changed, 319 insertions(+), 13 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
index 12561237247..f4cafc7e5be 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/dataformats/parquetAvro.json
@@ -18,6 +18,7 @@
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required":
false, "type": "string", "javaType": "java.lang.String", "deprecated": false,
"autowired": false, "secret": false, "description": "The id of this node" },
"compressionCodecName": { "index": 1, "kind": "attribute", "displayName":
"Compression Codec Name", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": "GZIP", "description": "Compression codec to use when
marshalling." },
- "unmarshalType": { "index": 2, "kind": "attribute", "displayName":
"Unmarshal Type", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"description": "Class to use when (un)marshalling. If omitted, parquet files
are converted into Avro's GenericRecords for unmarshalling and input objects
are assumed as GenericRecords for marshalling." }
+ "unmarshalType": { "index": 2, "kind": "attribute", "displayName":
"Unmarshal Type", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"description": "Class to use when (un)marshalling. If omitted, parquet files
are converted into Avro's GenericRecords for unmarshalling and input objects
are assumed as GenericRecords for marshalling." },
+ "lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load",
"required": false, "type": "boolean", "javaType": "java.lang.Boolean",
"deprecated": false, "autowired": false, "secret": false, "defaultValue":
false, "description": "Whether the unmarshalling should produce an iterator of
records or read all the records at once." }
}
}
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
index c4451799bde..048b6aa31e3 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/models/parquetAvro.json
@@ -15,6 +15,7 @@
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required":
false, "type": "string", "javaType": "java.lang.String", "deprecated": false,
"autowired": false, "secret": false, "description": "The id of this node" },
"compressionCodecName": { "index": 1, "kind": "attribute", "displayName":
"Compression Codec Name", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": "GZIP", "description": "Compression codec to use when
marshalling." },
- "unmarshalType": { "index": 2, "kind": "attribute", "displayName":
"Unmarshal Type", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"description": "Class to use when (un)marshalling. If omitted, parquet files
are converted into Avro's GenericRecords for unmarshalling and input objects
are assumed as GenericRecords for marshalling." }
+ "unmarshalType": { "index": 2, "kind": "attribute", "displayName":
"Unmarshal Type", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"description": "Class to use when (un)marshalling. If omitted, parquet files
are converted into Avro's GenericRecords for unmarshalling and input objects
are assumed as GenericRecords for marshalling." },
+ "lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load",
"required": false, "type": "boolean", "javaType": "java.lang.Boolean",
"deprecated": false, "autowired": false, "secret": false, "defaultValue":
false, "description": "Whether the unmarshalling should produce an iterator of
records or read all the records at once." }
}
}
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd
index f2f7982ac96..f754e021af4 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/schemas/camel-spring.xsd
@@ -8856,6 +8856,15 @@ Compression codec to use when marshalling. Default
value: GZIP
<![CDATA[
Class to use when (un)marshalling. If omitted, parquet files are converted
into Avro's GenericRecords for unmarshalling
and input objects are assumed as GenericRecords for marshalling.
+]]>
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute name="lazyLoad" type="xs:string">
+ <xs:annotation>
+ <xs:documentation xml:lang="en">
+<![CDATA[
+Whether the unmarshalling should produce an iterator of records or read all
the records at once. Default value: false
]]>
</xs:documentation>
</xs:annotation>
diff --git
a/components/camel-parquet-avro/src/generated/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormatConfigurer.java
b/components/camel-parquet-avro/src/generated/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormatConfigurer.java
index fef4c7f83f3..04ae2f99682 100644
---
a/components/camel-parquet-avro/src/generated/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormatConfigurer.java
+++
b/components/camel-parquet-avro/src/generated/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormatConfigurer.java
@@ -20,6 +20,8 @@ public class ParquetAvroDataFormatConfigurer extends
PropertyConfigurerSupport i
switch (ignoreCase ? name.toLowerCase() : name) {
case "unmarshaltype":
case "unmarshalType":
dataformat.setUnmarshalType(property(camelContext, java.lang.Class.class,
value)); return true;
+ case "lazyload":
+ case "lazyLoad": dataformat.setLazyLoad(property(camelContext,
boolean.class, value)); return true;
default: return false;
}
}
diff --git
a/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
b/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
index 12561237247..f4cafc7e5be 100644
---
a/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
+++
b/components/camel-parquet-avro/src/generated/resources/org/apache/camel/dataformat/parquet/avro/parquetAvro.json
@@ -18,6 +18,7 @@
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required":
false, "type": "string", "javaType": "java.lang.String", "deprecated": false,
"autowired": false, "secret": false, "description": "The id of this node" },
"compressionCodecName": { "index": 1, "kind": "attribute", "displayName":
"Compression Codec Name", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": "GZIP", "description": "Compression codec to use when
marshalling." },
- "unmarshalType": { "index": 2, "kind": "attribute", "displayName":
"Unmarshal Type", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"description": "Class to use when (un)marshalling. If omitted, parquet files
are converted into Avro's GenericRecords for unmarshalling and input objects
are assumed as GenericRecords for marshalling." }
+ "unmarshalType": { "index": 2, "kind": "attribute", "displayName":
"Unmarshal Type", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"description": "Class to use when (un)marshalling. If omitted, parquet files
are converted into Avro's GenericRecords for unmarshalling and input objects
are assumed as GenericRecords for marshalling." },
+ "lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load",
"required": false, "type": "boolean", "javaType": "java.lang.Boolean",
"deprecated": false, "autowired": false, "secret": false, "defaultValue":
false, "description": "Whether the unmarshalling should produce an iterator of
records or read all the records at once." }
}
}
diff --git
a/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
index b2aca2a7e6b..a4809da2bc8 100644
---
a/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
+++
b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormat.java
@@ -58,10 +58,14 @@ public class ParquetAvroDataFormat extends ServiceSupport
implements DataFormat,
private Class<?> unmarshalType;
+ private boolean lazyLoad;
+
+ @Override
public String getDataFormatName() {
return "parquetAvro";
}
+ @Override
public void marshal(Exchange exchange, Object graph, OutputStream stream)
throws Exception {
// marshal from the Java object or GenericRecord (graph) to the
parquet-avro type
Configuration conf = new Configuration();
@@ -103,9 +107,9 @@ public class ParquetAvroDataFormat extends ServiceSupport
implements DataFormat,
}
}
+ @Override
public Object unmarshal(Exchange exchange, InputStream stream) throws
Exception {
// unmarshal from the input stream of parquet-avro to Java object or
GenericRecord (graph)
- List<Object> parquetObjects = new ArrayList<>();
Configuration conf = new Configuration();
ParquetInputStream parquetInputStream = new ParquetInputStream(
@@ -119,18 +123,26 @@ public class ParquetAvroDataFormat extends ServiceSupport
implements DataFormat,
model = new ReflectData(unmarshalType.getClassLoader());
}
- try (ParquetReader<?> reader =
AvroParquetReader.builder(parquetInputStream)
+ ParquetReader.Builder<?> builder =
AvroParquetReader.builder(parquetInputStream)
.withDataModel(model)
.disableCompatibility() // always use this (since this is a
new project)
- .withConf(conf)
- .build()) {
- Object pojo;
- while ((pojo = type.cast(reader.read())) != null) {
- parquetObjects.add(pojo);
+ .withConf(conf);
+
+ if (lazyLoad) {
+ ParquetIterator<?> iterator = new
ParquetIterator<>(builder.build());
+ exchange.getExchangeExtension()
+ .addOnCompletion(new
ParquetUnmarshalOnCompletion(iterator));
+ return iterator;
+ } else {
+ try (ParquetReader<?> reader = builder.build()) {
+ List<Object> parquetObjects = new ArrayList<>();
+ Object pojo;
+ while ((pojo = type.cast(reader.read())) != null) {
+ parquetObjects.add(pojo);
+ }
+ return parquetObjects;
}
}
-
- return parquetObjects;
}
@Override
@@ -167,4 +179,19 @@ public class ParquetAvroDataFormat extends ServiceSupport
implements DataFormat,
this.unmarshalType = unmarshalType;
}
+ /**
+ * Indicates whether the unmarshalling should produce an iterator of
records or read all the records at once.
+ */
+ public boolean isLazyLoad() {
+ return lazyLoad;
+ }
+
+ /**
+ * Sets whether the unmarshalling should produce an iterator of records or
read all the records at once.
+ */
+ public ParquetAvroDataFormat setLazyLoad(boolean lazyLoad) {
+ this.lazyLoad = lazyLoad;
+ return this;
+ }
+
}
diff --git
a/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetIterator.java
b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetIterator.java
new file mode 100644
index 00000000000..6bac98b0196
--- /dev/null
+++
b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetIterator.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dataformat.parquet.avro;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.camel.RuntimeCamelException;
+import org.apache.parquet.hadoop.ParquetReader;
+
+public class ParquetIterator<T> implements Iterator<T>, Closeable {
+ private final ParquetReader<T> reader;
+ private T current;
+
+ public ParquetIterator(ParquetReader<T> reader) {
+ this.reader = reader;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (current == null) {
+ current = getNext();
+ }
+ return current != null;
+ }
+
+ @Override
+ public T next() {
+ T next = current;
+ current = null;
+ if (next == null) {
+ next = getNext();
+ if (next == null) {
+ throw new NoSuchElementException("No more items available");
+ }
+ }
+ return next;
+ }
+
+ private T getNext() {
+ try {
+ return reader.read();
+ } catch (IOException e) {
+ throw new RuntimeCamelException(e);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+}
diff --git
a/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetUnmarshalOnCompletion.java
b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetUnmarshalOnCompletion.java
new file mode 100644
index 00000000000..47ef6279f2f
--- /dev/null
+++
b/components/camel-parquet-avro/src/main/java/org/apache/camel/dataformat/parquet/avro/ParquetUnmarshalOnCompletion.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dataformat.parquet.avro;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.support.SynchronizationAdapter;
+
+public class ParquetUnmarshalOnCompletion extends SynchronizationAdapter {
+
+ private final Closeable closeable;
+
+ public ParquetUnmarshalOnCompletion(Closeable closeable) {
+ this.closeable = closeable;
+ }
+
+ @Override
+ public void onDone(Exchange exchange) {
+ try {
+ closeable.close();
+ } catch (IOException e) {
+ // ignore
+ }
+ }
+}
diff --git
a/components/camel-parquet-avro/src/test/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormatUnmarshalLazyLoadTest.java
b/components/camel-parquet-avro/src/test/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormatUnmarshalLazyLoadTest.java
new file mode 100644
index 00000000000..f532745dabd
--- /dev/null
+++
b/components/camel-parquet-avro/src/test/java/org/apache/camel/dataformat/parquet/avro/ParquetAvroDataFormatUnmarshalLazyLoadTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.dataformat.parquet.avro;
+
+import java.io.FileInputStream;
+import java.util.List;
+
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.generic.GenericData.Record;
+import org.apache.avro.generic.GenericRecordBuilder;
+import org.apache.camel.Exchange;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.test.junit5.CamelTestSupport;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class ParquetAvroDataFormatUnmarshalLazyLoadTest extends
CamelTestSupport {
+
+ @Test
+ public void testUnmarshalLazyLoad() throws Exception {
+
+ MockEndpoint mockResults = getMockEndpoint("mock:result");
+
+ mockResults.expectedMessageCount(3);
+ mockResults.message(0).body().isEqualTo(new Pojo(1, "airport"));
+ mockResults.message(1).body().isEqualTo(new Pojo(2, "penguin"));
+ mockResults.message(2).body().isEqualTo(new Pojo(3, "verb"));
+
+ template.sendBody("direct:start", new
FileInputStream("src/test/resources/example1.parquet"));
+
+ mockResults.assertIsSatisfied();
+ }
+
+ @Test
+ public void testUnmarshalLazyLoadNoUnmarshalType() throws Exception {
+ Schema schema = SchemaBuilder
+ .record("Pojo")
+ .fields()
+ .requiredString("data")
+ .requiredLong("id")
+ .endRecord();
+
+ Record expected1 = new GenericRecordBuilder(schema).set("data",
"airport").set("id", 1L).build();
+ Record expected2 = new GenericRecordBuilder(schema).set("data",
"penguin").set("id", 2L).build();
+ Record expected3 = new GenericRecordBuilder(schema).set("data",
"verb").set("id", 3L).build();
+
+ MockEndpoint mockResults =
getMockEndpoint("mock:resultNoUnmarshalType");
+
+ mockResults.expectedMessageCount(3);
+
+ template.sendBody("direct:startNoUnmarshalType", new
FileInputStream("src/test/resources/example1.parquet"));
+
+ mockResults.assertIsSatisfied();
+
+ List<Exchange> exchanges = mockResults.getExchanges();
+ assertEquals(0,
exchanges.get(0).getMessage().getBody(Record.class).compareTo(expected1));
+ assertEquals(0,
exchanges.get(1).getMessage().getBody(Record.class).compareTo(expected2));
+ assertEquals(0,
exchanges.get(2).getMessage().getBody(Record.class).compareTo(expected3));
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+
+ @Override
+ public void configure() throws Exception {
+ ParquetAvroDataFormat format = new ParquetAvroDataFormat()
+ .setLazyLoad(true);
+ format.setUnmarshalType(Pojo.class);
+
+ ParquetAvroDataFormat formatNoUnmarshalType = new
ParquetAvroDataFormat()
+ .setLazyLoad(true);
+
+ from("direct:start")
+ .unmarshal(format)
+ .split(body())
+ .to("mock:result");
+
+ from("direct:startNoUnmarshalType")
+ .unmarshal(formatNoUnmarshalType)
+ .split(body())
+ .to("mock:resultNoUnmarshalType");
+ }
+ };
+ }
+}
diff --git
a/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
b/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
index c4451799bde..048b6aa31e3 100644
---
a/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
+++
b/core/camel-core-model/src/generated/resources/org/apache/camel/model/dataformat/parquetAvro.json
@@ -15,6 +15,7 @@
"properties": {
"id": { "index": 0, "kind": "attribute", "displayName": "Id", "required":
false, "type": "string", "javaType": "java.lang.String", "deprecated": false,
"autowired": false, "secret": false, "description": "The id of this node" },
"compressionCodecName": { "index": 1, "kind": "attribute", "displayName":
"Compression Codec Name", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": "GZIP", "description": "Compression codec to use when
marshalling." },
- "unmarshalType": { "index": 2, "kind": "attribute", "displayName":
"Unmarshal Type", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"description": "Class to use when (un)marshalling. If omitted, parquet files
are converted into Avro's GenericRecords for unmarshalling and input objects
are assumed as GenericRecords for marshalling." }
+ "unmarshalType": { "index": 2, "kind": "attribute", "displayName":
"Unmarshal Type", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "autowired": false, "secret": false,
"description": "Class to use when (un)marshalling. If omitted, parquet files
are converted into Avro's GenericRecords for unmarshalling and input objects
are assumed as GenericRecords for marshalling." },
+ "lazyLoad": { "index": 3, "kind": "attribute", "displayName": "Lazy Load",
"required": false, "type": "boolean", "javaType": "java.lang.Boolean",
"deprecated": false, "autowired": false, "secret": false, "defaultValue":
false, "description": "Whether the unmarshalling should produce an iterator of
records or read all the records at once." }
}
}
diff --git
a/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
b/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
index 2524996256b..402bc819837 100644
---
a/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
+++
b/core/camel-core-model/src/main/java/org/apache/camel/model/dataformat/ParquetAvroDataFormat.java
@@ -44,6 +44,10 @@ public class ParquetAvroDataFormat extends
DataFormatDefinition {
@XmlAttribute(name = "unmarshalType")
private String unmarshalTypeName;
+ @XmlAttribute
+ @Metadata(javaType = "java.lang.Boolean")
+ private String lazyLoad;
+
public ParquetAvroDataFormat() {
super("parquetAvro");
}
@@ -58,11 +62,17 @@ public class ParquetAvroDataFormat extends
DataFormatDefinition {
this.unmarshalType = unmarshalType;
}
+ public ParquetAvroDataFormat(boolean lazyLoad) {
+ this();
+ setLazyLoad(Boolean.toString(lazyLoad));
+ }
+
private ParquetAvroDataFormat(Builder builder) {
this();
this.compressionCodecName = builder.compressionCodecName;
this.unmarshalTypeName = builder.unmarshalTypeName;
this.unmarshalType = builder.unmarshalType;
+ this.lazyLoad = builder.lazyLoad;
}
/**
@@ -99,6 +109,17 @@ public class ParquetAvroDataFormat extends
DataFormatDefinition {
return unmarshalTypeName;
}
+ public String getLazyLoad() {
+ return lazyLoad;
+ }
+
+ /**
+ * Whether the unmarshalling should produce an iterator of records or read
all the records at once.
+ */
+ public void setLazyLoad(String lazyLoad) {
+ this.lazyLoad = lazyLoad;
+ }
+
/**
* {@code Builder} is a specific builder for {@link ParquetAvroDataFormat}.
*/
@@ -108,6 +129,7 @@ public class ParquetAvroDataFormat extends
DataFormatDefinition {
private String compressionCodecName;
private Class<?> unmarshalType;
private String unmarshalTypeName;
+ private String lazyLoad;
/**
* Compression codec to use when marshalling.
@@ -133,6 +155,22 @@ public class ParquetAvroDataFormat extends
DataFormatDefinition {
return this;
}
+ /**
+ * Whether the unmarshalling should produce an iterator of records or
read all the records at once.
+ */
+ public Builder lazyLoad(String lazyLoad) {
+ this.lazyLoad = lazyLoad;
+ return this;
+ }
+
+ /**
+ * Whether the unmarshalling should produce an iterator of records or
read all the records at once.
+ */
+ public Builder lazyLoad(boolean lazyLoad) {
+ this.lazyLoad = Boolean.toString(lazyLoad);
+ return this;
+ }
+
@Override
public ParquetAvroDataFormat end() {
return new ParquetAvroDataFormat(this);
diff --git
a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java
b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java
index d633c037d78..84cf0cc12f8 100644
---
a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java
+++
b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/in/ModelParser.java
@@ -2450,6 +2450,7 @@ public class ModelParser extends BaseParser {
return doParse(new ParquetAvroDataFormat(), (def, key, val) -> {
switch (key) {
case "compressionCodecName": def.setCompressionCodecName(val);
break;
+ case "lazyLoad": def.setLazyLoad(val); break;
case "unmarshalType": def.setUnmarshalTypeName(val); break;
default: return identifiedTypeAttributeHandler().accept(def,
key, val);
}
diff --git
a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java
b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java
index 3c24894bdfd..f64fde80adf 100644
---
a/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java
+++
b/core/camel-xml-io/src/generated/java/org/apache/camel/xml/out/ModelWriter.java
@@ -3465,6 +3465,7 @@ public class ModelWriter extends BaseWriter {
doWriteIdentifiedTypeAttributes(def);
doWriteAttribute("compressionCodecName",
def.getCompressionCodecName());
doWriteAttribute("unmarshalType", def.getUnmarshalTypeName());
+ doWriteAttribute("lazyLoad", def.getLazyLoad());
endElement(name);
}
protected void doWriteProtobufDataFormat(
diff --git
a/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java
b/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java
index 6cbbea60a2e..603c2ed5ac2 100644
---
a/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java
+++
b/core/camel-yaml-io/src/generated/java/org/apache/camel/yaml/out/ModelWriter.java
@@ -3465,6 +3465,7 @@ public class ModelWriter extends BaseWriter {
doWriteIdentifiedTypeAttributes(def);
doWriteAttribute("compressionCodecName",
def.getCompressionCodecName());
doWriteAttribute("unmarshalType", def.getUnmarshalTypeName());
+ doWriteAttribute("lazyLoad", def.getLazyLoad());
endElement(name);
}
protected void doWriteProtobufDataFormat(
diff --git
a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java
b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java
index b7b30ade16a..b913c3f78eb 100644
---
a/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java
+++
b/dsl/camel-yaml-dsl/camel-yaml-dsl-deserializers/src/generated/java/org/apache/camel/dsl/yaml/deserializers/ModelDeserializers.java
@@ -10703,6 +10703,7 @@ public final class ModelDeserializers extends
YamlDeserializerSupport {
properties = {
@YamlProperty(name = "compressionCodecName", type =
"string", defaultValue = "GZIP", description = "Compression codec to use when
marshalling.", displayName = "Compression Codec Name"),
@YamlProperty(name = "id", type = "string", description =
"The id of this node", displayName = "Id"),
+ @YamlProperty(name = "lazyLoad", type = "boolean",
description = "Whether the unmarshalling should produce an iterator of records
or read all the records at once.", displayName = "Lazy Load"),
@YamlProperty(name = "unmarshalType", type = "string",
description = "Class to use when (un)marshalling. If omitted, parquet files are
converted into Avro's GenericRecords for unmarshalling and input objects are
assumed as GenericRecords for marshalling.", displayName = "Unmarshal Type")
}
)
@@ -10736,6 +10737,11 @@ public final class ModelDeserializers extends
YamlDeserializerSupport {
target.setId(val);
break;
}
+ case "lazyLoad": {
+ String val = asText(node);
+ target.setLazyLoad(val);
+ break;
+ }
case "unmarshalType": {
String val = asText(node);
target.setUnmarshalTypeName(val);
diff --git
a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json
b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json
index c33911e3e73..5593b355f78 100644
---
a/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json
+++
b/dsl/camel-yaml-dsl/camel-yaml-dsl/src/generated/resources/schema/camelYamlDsl.json
@@ -10685,6 +10685,11 @@
"title" : "Id",
"description" : "The id of this node"
},
+ "lazyLoad" : {
+ "type" : "boolean",
+ "title" : "Lazy Load",
+ "description" : "Whether the unmarshalling should produce an
iterator of records or read all the records at once."
+ },
"unmarshalType" : {
"type" : "string",
"title" : "Unmarshal Type",
