This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
The following commit(s) were added to refs/heads/main by this push:
new b0790956b Add Databricks source and sink Kamelet (#2672)
b0790956b is described below
commit b0790956b189d1eb216658bf293966faf41e442a
Author: Andrea Cosentino <[email protected]>
AuthorDate: Sat Dec 27 12:31:06 2025 +0100
Add Databricks source and sink Kamelet (#2672)
Signed-off-by: Andrea Cosentino <[email protected]>
---
kamelets/databricks-sink.kamelet.yaml | 103 +++++++++++++++++++
kamelets/databricks-source.kamelet.yaml | 114 +++++++++++++++++++++
.../kamelets/databricks-sink.kamelet.yaml | 103 +++++++++++++++++++
.../kamelets/databricks-source.kamelet.yaml | 114 +++++++++++++++++++++
4 files changed, 434 insertions(+)
diff --git a/kamelets/databricks-sink.kamelet.yaml
b/kamelets/databricks-sink.kamelet.yaml
new file mode 100644
index 000000000..fc3093807
--- /dev/null
+++ b/kamelets/databricks-sink.kamelet.yaml
@@ -0,0 +1,103 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: databricks-sink
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.17.0-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMC
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "SQL"
+ camel.apache.org/kamelet.namespace: "Database"
+ labels:
+ camel.apache.org/kamelet.type: "sink"
+spec:
+ definition:
+ title: "Databricks Sink"
+ description: Send data to a Databricks Database. This Kamelet expects a
JSON-formatted body. Use key:value pairs to map the JSON fields and parameters.
For Unity Catalog workspaces, specify catalog and schema parameters.
+ required:
+ - serverHostname
+ - httpPath
+ - accessToken
+ - query
+ type: object
+ properties:
+ serverHostname:
+ title: Server Hostname
+ description: The Databricks server hostname.
+ type: string
+ example: adb-1234567890123456.7.azuredatabricks.net
+ serverPort:
+ title: Server Port
+ description: The server port for the Databricks data source.
+ type: string
+ default: 443
+ httpPath:
+ title: HTTP Path
+ description: The HTTP path to the Databricks SQL Warehouse or cluster.
+ type: string
+ example: /sql/1.0/warehouses/abc123def456
+ accessToken:
+ title: Access Token
+ description: The personal access token to access Databricks.
+ type: string
+ format: password
+ x-descriptors:
+ - urn:camel:group:credentials
+ query:
+ title: Query
+ description: The query to execute against the Databricks Database.
+ type: string
+ example: 'INSERT INTO accounts (username,city) VALUES
(:#username,:#city)'
+ extraOptions:
+ title: Extra Options
+ description: Additional JDBC connection options (e.g.,
ConnCatalog=main;ConnSchema=default).
+ type: string
+ default: ''
+ types:
+ in:
+ mediaType: application/json
+ dependencies:
+ - "camel:jackson"
+ - "camel:kamelet"
+ - "camel:sql"
+ - "mvn:com.databricks:databricks-jdbc:3.0.7"
+ - "mvn:org.apache.commons:commons-dbcp2:2.14.0"
+ template:
+ beans:
+ - name: local-sql-databricks-sink
+ type: "#class:org.apache.camel.component.sql.SqlComponent"
+ properties:
+ autowiredEnabled: "false"
+ - name: dsBean
+ type: "#class:org.apache.commons.dbcp2.BasicDataSource"
+ properties:
+ password: '{{accessToken}}'
+ url:
'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}'
+ driverClassName: 'com.databricks.client.jdbc.Driver'
+ from:
+ uri: "kamelet:source"
+ steps:
+ - unmarshal:
+ json:
+ library: Jackson
+ - to:
+ uri: "{{local-sql-databricks-sink}}:{{query}}"
+ parameters:
+ dataSource: "#bean:{{dsBean}}"
diff --git a/kamelets/databricks-source.kamelet.yaml
b/kamelets/databricks-source.kamelet.yaml
new file mode 100644
index 000000000..29d05a263
--- /dev/null
+++ b/kamelets/databricks-source.kamelet.yaml
@@ -0,0 +1,114 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: databricks-source
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.17.0-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMC
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "SQL"
+ camel.apache.org/kamelet.namespace: "Database"
+ labels:
+ camel.apache.org/kamelet.type: "source"
+spec:
+ definition:
+ title: "Databricks Source"
+ description: Query data from a Databricks Database. For Unity Catalog
workspaces, specify catalog and schema parameters.
+ required:
+ - serverHostname
+ - httpPath
+ - accessToken
+ - query
+ type: object
+ properties:
+ serverHostname:
+ title: Server Hostname
+ description: The Databricks server hostname.
+ type: string
+ example: adb-1234567890123456.7.azuredatabricks.net
+ serverPort:
+ title: Server Port
+ description: The server port for the Databricks data source.
+ type: string
+ default: 443
+ httpPath:
+ title: HTTP Path
+ description: The HTTP path to the Databricks SQL Warehouse or cluster.
+ type: string
+ example: /sql/1.0/warehouses/abc123def456
+ accessToken:
+ title: Access Token
+ description: The personal access token to access Databricks.
+ type: string
+ format: password
+ x-descriptors:
+ - urn:camel:group:credentials
+ query:
+ title: Query
+ description: The query to execute against the Databricks Database.
+ type: string
+ example: 'SELECT * FROM accounts'
+ extraOptions:
+ title: Extra Options
+ description: Additional JDBC connection options (e.g.,
ConnCatalog=main;ConnSchema=default).
+ type: string
+ default: ''
+ consumedQuery:
+ title: Consumed Query
+ description: A query to run on a tuple consumed.
+ type: string
+ example: 'DELETE FROM accounts where user_id = :#user_id'
+ delay:
+ title: Delay
+ description: The number of milliseconds before the next poll from the
Databricks database.
+ type: integer
+ default: 500
+ types:
+ out:
+ mediaType: application/json
+ dependencies:
+ - "camel:jackson"
+ - "camel:kamelet"
+ - "camel:sql"
+ - "mvn:com.databricks:databricks-jdbc:3.0.7"
+ - "mvn:org.apache.commons:commons-dbcp2:2.14.0"
+ template:
+ beans:
+ - name: local-sql-databricks-source
+ type: "#class:org.apache.camel.component.sql.SqlComponent"
+ properties:
+ autowiredEnabled: "false"
+ - name: dsBean
+ type: "#class:org.apache.commons.dbcp2.BasicDataSource"
+ properties:
+ password: '{{accessToken}}'
+ url:
'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}'
+ driverClassName: 'com.databricks.client.jdbc.Driver'
+ from:
+ uri: "{{local-sql-databricks-source}}:{{query}}"
+ parameters:
+ dataSource: "#bean:{{dsBean}}"
+ onConsume: "{{?consumedQuery}}"
+ delay: "{{delay}}"
+ steps:
+ - marshal:
+ json:
+ library: Jackson
+ - to: "kamelet:sink"
diff --git
a/library/camel-kamelets/src/main/resources/kamelets/databricks-sink.kamelet.yaml
b/library/camel-kamelets/src/main/resources/kamelets/databricks-sink.kamelet.yaml
new file mode 100644
index 000000000..fc3093807
--- /dev/null
+++
b/library/camel-kamelets/src/main/resources/kamelets/databricks-sink.kamelet.yaml
@@ -0,0 +1,103 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: databricks-sink
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.17.0-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMC
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "SQL"
+ camel.apache.org/kamelet.namespace: "Database"
+ labels:
+ camel.apache.org/kamelet.type: "sink"
+spec:
+ definition:
+ title: "Databricks Sink"
+ description: Send data to a Databricks Database. This Kamelet expects a
JSON-formatted body. Use key:value pairs to map the JSON fields and parameters.
For Unity Catalog workspaces, specify catalog and schema parameters.
+ required:
+ - serverHostname
+ - httpPath
+ - accessToken
+ - query
+ type: object
+ properties:
+ serverHostname:
+ title: Server Hostname
+ description: The Databricks server hostname.
+ type: string
+ example: adb-1234567890123456.7.azuredatabricks.net
+ serverPort:
+ title: Server Port
+ description: The server port for the Databricks data source.
+ type: string
+ default: 443
+ httpPath:
+ title: HTTP Path
+ description: The HTTP path to the Databricks SQL Warehouse or cluster.
+ type: string
+ example: /sql/1.0/warehouses/abc123def456
+ accessToken:
+ title: Access Token
+ description: The personal access token to access Databricks.
+ type: string
+ format: password
+ x-descriptors:
+ - urn:camel:group:credentials
+ query:
+ title: Query
+ description: The query to execute against the Databricks Database.
+ type: string
+ example: 'INSERT INTO accounts (username,city) VALUES
(:#username,:#city)'
+ extraOptions:
+ title: Extra Options
+ description: Additional JDBC connection options (e.g.,
ConnCatalog=main;ConnSchema=default).
+ type: string
+ default: ''
+ types:
+ in:
+ mediaType: application/json
+ dependencies:
+ - "camel:jackson"
+ - "camel:kamelet"
+ - "camel:sql"
+ - "mvn:com.databricks:databricks-jdbc:3.0.7"
+ - "mvn:org.apache.commons:commons-dbcp2:2.14.0"
+ template:
+ beans:
+ - name: local-sql-databricks-sink
+ type: "#class:org.apache.camel.component.sql.SqlComponent"
+ properties:
+ autowiredEnabled: "false"
+ - name: dsBean
+ type: "#class:org.apache.commons.dbcp2.BasicDataSource"
+ properties:
+ password: '{{accessToken}}'
+ url:
'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}'
+ driverClassName: 'com.databricks.client.jdbc.Driver'
+ from:
+ uri: "kamelet:source"
+ steps:
+ - unmarshal:
+ json:
+ library: Jackson
+ - to:
+ uri: "{{local-sql-databricks-sink}}:{{query}}"
+ parameters:
+ dataSource: "#bean:{{dsBean}}"
diff --git
a/library/camel-kamelets/src/main/resources/kamelets/databricks-source.kamelet.yaml
b/library/camel-kamelets/src/main/resources/kamelets/databricks-source.kamelet.yaml
new file mode 100644
index 000000000..29d05a263
--- /dev/null
+++
b/library/camel-kamelets/src/main/resources/kamelets/databricks-source.kamelet.yaml
@@ -0,0 +1,114 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: databricks-source
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.17.0-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMC
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "SQL"
+ camel.apache.org/kamelet.namespace: "Database"
+ labels:
+ camel.apache.org/kamelet.type: "source"
+spec:
+ definition:
+ title: "Databricks Source"
+ description: Query data from a Databricks Database. For Unity Catalog
workspaces, specify catalog and schema parameters.
+ required:
+ - serverHostname
+ - httpPath
+ - accessToken
+ - query
+ type: object
+ properties:
+ serverHostname:
+ title: Server Hostname
+ description: The Databricks server hostname.
+ type: string
+ example: adb-1234567890123456.7.azuredatabricks.net
+ serverPort:
+ title: Server Port
+ description: The server port for the Databricks data source.
+ type: string
+ default: 443
+ httpPath:
+ title: HTTP Path
+ description: The HTTP path to the Databricks SQL Warehouse or cluster.
+ type: string
+ example: /sql/1.0/warehouses/abc123def456
+ accessToken:
+ title: Access Token
+ description: The personal access token to access Databricks.
+ type: string
+ format: password
+ x-descriptors:
+ - urn:camel:group:credentials
+ query:
+ title: Query
+ description: The query to execute against the Databricks Database.
+ type: string
+ example: 'SELECT * FROM accounts'
+ extraOptions:
+ title: Extra Options
+ description: Additional JDBC connection options (e.g.,
ConnCatalog=main;ConnSchema=default).
+ type: string
+ default: ''
+ consumedQuery:
+ title: Consumed Query
+ description: A query to run on a tuple consumed.
+ type: string
+ example: 'DELETE FROM accounts where user_id = :#user_id'
+ delay:
+ title: Delay
+ description: The number of milliseconds before the next poll from the
Databricks database.
+ type: integer
+ default: 500
+ types:
+ out:
+ mediaType: application/json
+ dependencies:
+ - "camel:jackson"
+ - "camel:kamelet"
+ - "camel:sql"
+ - "mvn:com.databricks:databricks-jdbc:3.0.7"
+ - "mvn:org.apache.commons:commons-dbcp2:2.14.0"
+ template:
+ beans:
+ - name: local-sql-databricks-source
+ type: "#class:org.apache.camel.component.sql.SqlComponent"
+ properties:
+ autowiredEnabled: "false"
+ - name: dsBean
+ type: "#class:org.apache.commons.dbcp2.BasicDataSource"
+ properties:
+ password: '{{accessToken}}'
+ url:
'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}'
+ driverClassName: 'com.databricks.client.jdbc.Driver'
+ from:
+ uri: "{{local-sql-databricks-source}}:{{query}}"
+ parameters:
+ dataSource: "#bean:{{dsBean}}"
+ onConsume: "{{?consumedQuery}}"
+ delay: "{{delay}}"
+ steps:
+ - marshal:
+ json:
+ library: Jackson
+ - to: "kamelet:sink"
