This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 57215927777f79d6611246e30d0dd077d22af717
Author: Andrea Cosentino <anco...@gmail.com>
AuthorDate: Tue Apr 6 08:37:39 2021 +0200
CAMEL-16185 - AWS S3: improve multipart support - streaming upload
---
.../resources/org/apache/camel/catalog/components/aws2-s3.json | 6 ++++--
.../resources/org/apache/camel/catalog/docs/aws2-s3-component.adoc | 6 +++---
docs/components/modules/ROOT/pages/aws2-s3-component.adoc | 6 +++---
3 files changed, 10 insertions(+), 8 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/aws2-s3.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/aws2-s3.json
index feced29..584eec8 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/aws2-s3.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/aws2-s3.json
@@ -62,7 +62,8 @@
"operation": { "kind": "property", "displayName": "Operation", "group":
"producer", "label": "producer", "required": false, "type": "object",
"javaType": "org.apache.camel.component.aws2.s3.AWS2S3Operations", "enum": [
"copyObject", "listObjects", "deleteObject", "deleteBucket", "listBuckets",
"getObject", "getObjectRange" ], "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
[...]
"partSize": { "kind": "property", "displayName": "Part Size", "group":
"producer", "label": "producer", "required": false, "type": "integer",
"javaType": "long", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": 26214400, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "Setup the partSize which is used in multi part
upload, the default size is 25M." },
"storageClass": { "kind": "property", "displayName": "Storage Class",
"group": "producer", "label": "producer", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "The storage class to set in the
com.amazonaws.services.s3.model.PutObjectRequest request." },
- "streamMode": { "kind": "property", "displayName": "Stream Mode", "group":
"producer", "label": "producer", "required": false, "type": "boolean",
"javaType": "boolean", "deprecated": false, "autowired": false, "secret":
false, "defaultValue": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "When stream mode is true the upload to bucket
will be done in streaming" },
+ "streamingUploadMode": { "kind": "property", "displayName": "Streaming
Upload Mode", "group": "producer", "label": "producer", "required": false,
"type": "boolean", "javaType": "boolean", "deprecated": false, "autowired":
false, "secret": false, "defaultValue": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "When stream mode is true the upload to bucket
will be done in streaming" },
+ "streamingUploadTimeout": { "kind": "property", "displayName": "Streaming
Upload Timeout", "group": "producer", "label": "producer", "required": false,
"type": "integer", "javaType": "long", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "While streaming upload mode is true, this
option set the timeout to complete upload" },
"awsKMSKeyId": { "kind": "property", "displayName": "Aws KMSKey Id",
"group": "producer (advanced)", "label": "producer,advanced", "required":
false, "type": "string", "javaType": "java.lang.String", "deprecated": false,
"autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "Define the id of KMS key to use in case KMS is
enabled" },
"useAwsKMS": { "kind": "property", "displayName": "Use Aws KMS", "group":
"producer (advanced)", "label": "producer,advanced", "required": false, "type":
"boolean", "javaType": "boolean", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "Define if KMS must be used or not" },
"useCustomerKey": { "kind": "property", "displayName": "Use Customer Key",
"group": "producer (advanced)", "label": "producer,advanced", "required":
false, "type": "boolean", "javaType": "boolean", "deprecated": false,
"autowired": false, "secret": false, "defaultValue": false,
"configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration",
"configurationField": "configuration", "description": "Define if Customer Key
must be used or not" },
@@ -117,7 +118,8 @@
"operation": { "kind": "parameter", "displayName": "Operation", "group":
"producer", "label": "producer", "required": false, "type": "object",
"javaType": "org.apache.camel.component.aws2.s3.AWS2S3Operations", "enum": [
"copyObject", "listObjects", "deleteObject", "deleteBucket", "listBuckets",
"getObject", "getObjectRange" ], "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
[...]
"partSize": { "kind": "parameter", "displayName": "Part Size", "group":
"producer", "label": "producer", "required": false, "type": "integer",
"javaType": "long", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": 26214400, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "Setup the partSize which is used in multi part
upload, the default size is 25M." },
"storageClass": { "kind": "parameter", "displayName": "Storage Class",
"group": "producer", "label": "producer", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "The storage class to set in the
com.amazonaws.services.s3.model.PutObjectRequest request." },
- "streamMode": { "kind": "parameter", "displayName": "Stream Mode",
"group": "producer", "label": "producer", "required": false, "type": "boolean",
"javaType": "boolean", "deprecated": false, "autowired": false, "secret":
false, "defaultValue": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "When stream mode is true the upload to bucket
will be done in streaming" },
+ "streamingUploadMode": { "kind": "parameter", "displayName": "Streaming
Upload Mode", "group": "producer", "label": "producer", "required": false,
"type": "boolean", "javaType": "boolean", "deprecated": false, "autowired":
false, "secret": false, "defaultValue": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "When stream mode is true the upload to bucket
will be done in streaming" },
+ "streamingUploadTimeout": { "kind": "parameter", "displayName": "Streaming
Upload Timeout", "group": "producer", "label": "producer", "required": false,
"type": "integer", "javaType": "long", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "While streaming upload mode is true, this
option set the timeout to complete upload" },
"awsKMSKeyId": { "kind": "parameter", "displayName": "Aws KMSKey Id",
"group": "producer (advanced)", "label": "producer,advanced", "required":
false, "type": "string", "javaType": "java.lang.String", "deprecated": false,
"autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "Define the id of KMS key to use in case KMS is
enabled" },
"useAwsKMS": { "kind": "parameter", "displayName": "Use Aws KMS", "group":
"producer (advanced)", "label": "producer,advanced", "required": false, "type":
"boolean", "javaType": "boolean", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": false, "configurationClass":
"org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField":
"configuration", "description": "Define if KMS must be used or not" },
"useCustomerKey": { "kind": "parameter", "displayName": "Use Customer
Key", "group": "producer (advanced)", "label": "producer,advanced", "required":
false, "type": "boolean", "javaType": "boolean", "deprecated": false,
"autowired": false, "secret": false, "defaultValue": false,
"configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration",
"configurationField": "configuration", "description": "Define if Customer Key
must be used or not" },
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/aws2-s3-component.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/aws2-s3-component.adoc
index dc3fd8f..4489346 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/aws2-s3-component.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/aws2-s3-component.adoc
@@ -555,7 +555,7 @@ This operation will return an S3Object instance related to
the camelKey object i
This operation will return a download link url for the file camel-key in the
bucket mycamelbucket and region region
-== Stream mode
+== Streaming Upload mode
With the stream mode enabled users will be able to upload data to S3 without
knowing ahead of time the dimension of the data, by leveraging multipart upload.
The upload will be completed when: the batchSize has been completed or the
batchMessageNumber has been reached.
@@ -567,11 +567,11 @@ As an example:
--------------------------------------------------------------------------------
from(kafka("topic1").brokers("localhost:9092"))
.log("Kafka Message is: ${body}")
-
.to(aws2S3("camel-bucket").streamMode(true).batchMessageNumber(25).namingStrategy(AWS2S3EndpointBuilderFactory.AWSS3NamingStrategyEnum.progressive).keyName("{{kafkaTopic1}}/{{kafkaTopic1}}.txt"));
+
.to(aws2S3("camel-bucket").streamingUploadMode(true).batchMessageNumber(25).namingStrategy(AWS2S3EndpointBuilderFactory.AWSS3NamingStrategyEnum.progressive).keyName("{{kafkaTopic1}}/{{kafkaTopic1}}.txt"));
from(kafka("topic2").brokers("localhost:9092"))
.log("Kafka Message is: ${body}")
-
.to(aws2S3("camel-bucket").streamMode(true).batchMessageNumber(25).namingStrategy(AWS2S3EndpointBuilderFactory.AWSS3NamingStrategyEnum.progressive).keyName("{{kafkaTopic2}}/{{kafkaTopic2}}.txt"));
+
.to(aws2S3("camel-bucket").streamingUploadMode(true).batchMessageNumber(25).namingStrategy(AWS2S3EndpointBuilderFactory.AWSS3NamingStrategyEnum.progressive).keyName("{{kafkaTopic2}}/{{kafkaTopic2}}.txt"));
--------------------------------------------------------------------------------
The default size for a batch is 1 Mb, but you can adjust it according to your
requirements.
diff --git a/docs/components/modules/ROOT/pages/aws2-s3-component.adoc
b/docs/components/modules/ROOT/pages/aws2-s3-component.adoc
index f20ba1f..c0a137d 100644
--- a/docs/components/modules/ROOT/pages/aws2-s3-component.adoc
+++ b/docs/components/modules/ROOT/pages/aws2-s3-component.adoc
@@ -557,7 +557,7 @@ This operation will return an S3Object instance related to
the camelKey object i
This operation will return a download link url for the file camel-key in the
bucket mycamelbucket and region region
-== Stream mode
+== Streaming Upload mode
With the stream mode enabled users will be able to upload data to S3 without
knowing ahead of time the dimension of the data, by leveraging multipart upload.
The upload will be completed when: the batchSize has been completed or the
batchMessageNumber has been reached.
@@ -569,11 +569,11 @@ As an example:
--------------------------------------------------------------------------------
from(kafka("topic1").brokers("localhost:9092"))
.log("Kafka Message is: ${body}")
-
.to(aws2S3("camel-bucket").streamMode(true).batchMessageNumber(25).namingStrategy(AWS2S3EndpointBuilderFactory.AWSS3NamingStrategyEnum.progressive).keyName("{{kafkaTopic1}}/{{kafkaTopic1}}.txt"));
+
.to(aws2S3("camel-bucket").streamingUploadMode(true).batchMessageNumber(25).namingStrategy(AWS2S3EndpointBuilderFactory.AWSS3NamingStrategyEnum.progressive).keyName("{{kafkaTopic1}}/{{kafkaTopic1}}.txt"));
from(kafka("topic2").brokers("localhost:9092"))
.log("Kafka Message is: ${body}")
-
.to(aws2S3("camel-bucket").streamMode(true).batchMessageNumber(25).namingStrategy(AWS2S3EndpointBuilderFactory.AWSS3NamingStrategyEnum.progressive).keyName("{{kafkaTopic2}}/{{kafkaTopic2}}.txt"));
+
.to(aws2S3("camel-bucket").streamingUploadMode(true).batchMessageNumber(25).namingStrategy(AWS2S3EndpointBuilderFactory.AWSS3NamingStrategyEnum.progressive).keyName("{{kafkaTopic2}}/{{kafkaTopic2}}.txt"));
--------------------------------------------------------------------------------
The default size for a batch is 1 Mb, but you can adjust it according to your
requirements.
