This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch aws-s3-to-kafka
in repository https://gitbox.apache.org/repos/asf/camel-k-examples.git
commit aede39929d9c488ae387ddcce28df6050702e9b9
Author: Andrea Cosentino <anco...@gmail.com>
AuthorDate: Fri Jul 2 15:01:45 2021 +0200
Added AWS S3 to Kafka with Timestamp Router Example
---
.../README.md | 22 ++++++++++++
.../flow-binding.yaml | 40 ++++++++++++++++++++++
.../log-sink.kamelet.yaml | 22 ++++++++++++
3 files changed, 84 insertions(+)
diff --git a/kamelets/aws-s3-to-kafka-with-timestamp-router/README.md
b/kamelets/aws-s3-to-kafka-with-timestamp-router/README.md
new file mode 100644
index 0000000..a83a955
--- /dev/null
+++ b/kamelets/aws-s3-to-kafka-with-timestamp-router/README.md
@@ -0,0 +1,22 @@
+# AWS S3 to Kafka with Timestamp Router
+
+- Use the quickstart for https://strimzi.io/quickstarts/ and follow the
minikube guide.
+
+- Install camel-k on the kafka namespalce
+
+- Open the flow-binding.yaml file and insert the correct credentials for AWS
S3 account and the bucket name.
+
+- The Log Sink Kamelet is not available out of the box in 1.5.0 Camel-K
release so you'll have to install it before installing the flow binding.
+
+- Set the correct credentials for S3 in the flow-binding.yaml
+
+- Run the following commands
+
+ - kubectl apply -f log-sink.kamelet.yaml -n kafka
+ - kubectl apply -f flow-binding.yaml -n kafka
+
+- Check logs
+
+ - kamel logs s3-to-kafka-with-timestamp-router -n kafka
+
+You should see the file content from AWS S3 logged and appearing in a topic
based on the Last modified metadata field of the S3 file consumed. The
granularity of the topic names is minutes.
diff --git a/kamelets/aws-s3-to-kafka-with-timestamp-router/flow-binding.yaml
b/kamelets/aws-s3-to-kafka-with-timestamp-router/flow-binding.yaml
new file mode 100644
index 0000000..a6b712d
--- /dev/null
+++ b/kamelets/aws-s3-to-kafka-with-timestamp-router/flow-binding.yaml
@@ -0,0 +1,40 @@
+apiVersion: camel.apache.org/v1alpha1
+kind: KameletBinding
+metadata:
+ name: s3-to-kafka-with-timestamp-router
+spec:
+ source:
+ ref:
+ kind: Kamelet
+ apiVersion: camel.apache.org/v1alpha1
+ name: aws-s3-source
+ properties:
+ bucketNameOrArn: 'camel-kafka-connector'
+ accessKey: 'access'
+ secretKey: 'secrets'
+ region: 'reg'
+ steps:
+ - ref:
+ kind: Kamelet
+ apiVersion: camel.apache.org/v1alpha1
+ name: log-sink
+ - ref:
+ kind: Kamelet
+ apiVersion: camel.apache.org/v1alpha1
+ name: timestamp-router-action
+ properties:
+ topicFormat: 's3-$[timestamp]'
+ timestampFormat: 'YYYY-MM-dd-hh-mm'
+ timestampHeaderName: 'CamelAwsS3LastModified'
+ - ref:
+ kind: Kamelet
+ apiVersion: camel.apache.org/v1alpha1
+ name: log-sink
+ sink:
+ ref:
+ kind: Kamelet
+ apiVersion: camel.apache.org/v1alpha1
+ name: kafka-not-secured-sink
+ properties:
+ brokers: 'my-cluster-kafka-bootstrap:9092'
+ topic: 'test-topic'
diff --git
a/kamelets/aws-s3-to-kafka-with-timestamp-router/log-sink.kamelet.yaml
b/kamelets/aws-s3-to-kafka-with-timestamp-router/log-sink.kamelet.yaml
new file mode 100755
index 0000000..a48fb41
--- /dev/null
+++ b/kamelets/aws-s3-to-kafka-with-timestamp-router/log-sink.kamelet.yaml
@@ -0,0 +1,22 @@
+apiVersion: camel.apache.org/v1alpha1
+kind: Kamelet
+metadata:
+ name: log-sink
+ annotations:
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNDAgMjQwIj48ZGVmcz48bGluZWFyR3JhZGllbnQgaWQ9ImEiIHgxPSIuNjY3IiB4Mj0iLjQxNyIgeTE9Ii4xNjciIHkyPSIuNzUiPjxzdG9wIG9mZnNldD0iMCIgc3RvcC1jb2xvcj0iIzM3YWVlMiIvPjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzFlOTZjOCIvPjwvbGluZWFyR3JhZGllbnQ+PGxpbmVhckdyYWRpZW50IGlkPSJiIiB4MT0iLjY2IiB4Mj0iLjg1MSIgeTE9Ii40MzciIHkyPSIuODAyIj48c3RvcCBvZmZzZXQ9IjAiIHN0b3AtY29sb3I9IiNlZmY3Zm
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ labels:
+ camel.apache.org/kamelet.type: "sink"
+ camel.apache.org/kamelet.group: "Log"
+spec:
+ definition:
+ title: "Log Sink"
+ description: |-
+ Log something
+ type: object
+ flow:
+ from:
+ uri: "kamelet:source"
+ steps:
+ - to:
+ uri: "log:info?showAll=true"
