This is an automated email from the ASF dual-hosted git repository. acosentino pushed a commit to branch databrick-source-sink in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit 18eefb9e542a160136f79dcdae7e357a0382e23f Author: Andrea Cosentino <[email protected]> AuthorDate: Sat Dec 27 12:26:08 2025 +0100 Add Databricks source and sink Kamelet Signed-off-by: Andrea Cosentino <[email protected]> --- kamelets/databricks-sink.kamelet.yaml | 103 +++++++++++++++++++ kamelets/databricks-source.kamelet.yaml | 114 +++++++++++++++++++++ .../kamelets/databricks-sink.kamelet.yaml | 103 +++++++++++++++++++ .../kamelets/databricks-source.kamelet.yaml | 114 +++++++++++++++++++++ 4 files changed, 434 insertions(+) diff --git a/kamelets/databricks-sink.kamelet.yaml b/kamelets/databricks-sink.kamelet.yaml new file mode 100644 index 000000000..fc3093807 --- /dev/null +++ b/kamelets/databricks-sink.kamelet.yaml @@ -0,0 +1,103 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: databricks-sink + annotations: + camel.apache.org/kamelet.support.level: "Preview" + camel.apache.org/catalog.version: "4.17.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMC [...] + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "SQL" + camel.apache.org/kamelet.namespace: "Database" + labels: + camel.apache.org/kamelet.type: "sink" +spec: + definition: + title: "Databricks Sink" + description: Send data to a Databricks Database. This Kamelet expects a JSON-formatted body. Use key:value pairs to map the JSON fields and parameters. For Unity Catalog workspaces, specify catalog and schema parameters. + required: + - serverHostname + - httpPath + - accessToken + - query + type: object + properties: + serverHostname: + title: Server Hostname + description: The Databricks server hostname. + type: string + example: adb-1234567890123456.7.azuredatabricks.net + serverPort: + title: Server Port + description: The server port for the Databricks data source. + type: string + default: 443 + httpPath: + title: HTTP Path + description: The HTTP path to the Databricks SQL Warehouse or cluster. + type: string + example: /sql/1.0/warehouses/abc123def456 + accessToken: + title: Access Token + description: The personal access token to access Databricks. + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + query: + title: Query + description: The query to execute against the Databricks Database. + type: string + example: 'INSERT INTO accounts (username,city) VALUES (:#username,:#city)' + extraOptions: + title: Extra Options + description: Additional JDBC connection options (e.g., ConnCatalog=main;ConnSchema=default). + type: string + default: '' + types: + in: + mediaType: application/json + dependencies: + - "camel:jackson" + - "camel:kamelet" + - "camel:sql" + - "mvn:com.databricks:databricks-jdbc:3.0.7" + - "mvn:org.apache.commons:commons-dbcp2:2.14.0" + template: + beans: + - name: local-sql-databricks-sink + type: "#class:org.apache.camel.component.sql.SqlComponent" + properties: + autowiredEnabled: "false" + - name: dsBean + type: "#class:org.apache.commons.dbcp2.BasicDataSource" + properties: + password: '{{accessToken}}' + url: 'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}' + driverClassName: 'com.databricks.client.jdbc.Driver' + from: + uri: "kamelet:source" + steps: + - unmarshal: + json: + library: Jackson + - to: + uri: "{{local-sql-databricks-sink}}:{{query}}" + parameters: + dataSource: "#bean:{{dsBean}}" diff --git a/kamelets/databricks-source.kamelet.yaml b/kamelets/databricks-source.kamelet.yaml new file mode 100644 index 000000000..29d05a263 --- /dev/null +++ b/kamelets/databricks-source.kamelet.yaml @@ -0,0 +1,114 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: databricks-source + annotations: + camel.apache.org/kamelet.support.level: "Preview" + camel.apache.org/catalog.version: "4.17.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMC [...] + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "SQL" + camel.apache.org/kamelet.namespace: "Database" + labels: + camel.apache.org/kamelet.type: "source" +spec: + definition: + title: "Databricks Source" + description: Query data from a Databricks Database. For Unity Catalog workspaces, specify catalog and schema parameters. + required: + - serverHostname + - httpPath + - accessToken + - query + type: object + properties: + serverHostname: + title: Server Hostname + description: The Databricks server hostname. + type: string + example: adb-1234567890123456.7.azuredatabricks.net + serverPort: + title: Server Port + description: The server port for the Databricks data source. + type: string + default: 443 + httpPath: + title: HTTP Path + description: The HTTP path to the Databricks SQL Warehouse or cluster. + type: string + example: /sql/1.0/warehouses/abc123def456 + accessToken: + title: Access Token + description: The personal access token to access Databricks. + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + query: + title: Query + description: The query to execute against the Databricks Database. + type: string + example: 'SELECT * FROM accounts' + extraOptions: + title: Extra Options + description: Additional JDBC connection options (e.g., ConnCatalog=main;ConnSchema=default). + type: string + default: '' + consumedQuery: + title: Consumed Query + description: A query to run on a tuple consumed. + type: string + example: 'DELETE FROM accounts where user_id = :#user_id' + delay: + title: Delay + description: The number of milliseconds before the next poll from the Databricks database. + type: integer + default: 500 + types: + out: + mediaType: application/json + dependencies: + - "camel:jackson" + - "camel:kamelet" + - "camel:sql" + - "mvn:com.databricks:databricks-jdbc:3.0.7" + - "mvn:org.apache.commons:commons-dbcp2:2.14.0" + template: + beans: + - name: local-sql-databricks-source + type: "#class:org.apache.camel.component.sql.SqlComponent" + properties: + autowiredEnabled: "false" + - name: dsBean + type: "#class:org.apache.commons.dbcp2.BasicDataSource" + properties: + password: '{{accessToken}}' + url: 'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}' + driverClassName: 'com.databricks.client.jdbc.Driver' + from: + uri: "{{local-sql-databricks-source}}:{{query}}" + parameters: + dataSource: "#bean:{{dsBean}}" + onConsume: "{{?consumedQuery}}" + delay: "{{delay}}" + steps: + - marshal: + json: + library: Jackson + - to: "kamelet:sink" diff --git a/library/camel-kamelets/src/main/resources/kamelets/databricks-sink.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/databricks-sink.kamelet.yaml new file mode 100644 index 000000000..fc3093807 --- /dev/null +++ b/library/camel-kamelets/src/main/resources/kamelets/databricks-sink.kamelet.yaml @@ -0,0 +1,103 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: databricks-sink + annotations: + camel.apache.org/kamelet.support.level: "Preview" + camel.apache.org/catalog.version: "4.17.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMC [...] + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "SQL" + camel.apache.org/kamelet.namespace: "Database" + labels: + camel.apache.org/kamelet.type: "sink" +spec: + definition: + title: "Databricks Sink" + description: Send data to a Databricks Database. This Kamelet expects a JSON-formatted body. Use key:value pairs to map the JSON fields and parameters. For Unity Catalog workspaces, specify catalog and schema parameters. + required: + - serverHostname + - httpPath + - accessToken + - query + type: object + properties: + serverHostname: + title: Server Hostname + description: The Databricks server hostname. + type: string + example: adb-1234567890123456.7.azuredatabricks.net + serverPort: + title: Server Port + description: The server port for the Databricks data source. + type: string + default: 443 + httpPath: + title: HTTP Path + description: The HTTP path to the Databricks SQL Warehouse or cluster. + type: string + example: /sql/1.0/warehouses/abc123def456 + accessToken: + title: Access Token + description: The personal access token to access Databricks. + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + query: + title: Query + description: The query to execute against the Databricks Database. + type: string + example: 'INSERT INTO accounts (username,city) VALUES (:#username,:#city)' + extraOptions: + title: Extra Options + description: Additional JDBC connection options (e.g., ConnCatalog=main;ConnSchema=default). + type: string + default: '' + types: + in: + mediaType: application/json + dependencies: + - "camel:jackson" + - "camel:kamelet" + - "camel:sql" + - "mvn:com.databricks:databricks-jdbc:3.0.7" + - "mvn:org.apache.commons:commons-dbcp2:2.14.0" + template: + beans: + - name: local-sql-databricks-sink + type: "#class:org.apache.camel.component.sql.SqlComponent" + properties: + autowiredEnabled: "false" + - name: dsBean + type: "#class:org.apache.commons.dbcp2.BasicDataSource" + properties: + password: '{{accessToken}}' + url: 'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}' + driverClassName: 'com.databricks.client.jdbc.Driver' + from: + uri: "kamelet:source" + steps: + - unmarshal: + json: + library: Jackson + - to: + uri: "{{local-sql-databricks-sink}}:{{query}}" + parameters: + dataSource: "#bean:{{dsBean}}" diff --git a/library/camel-kamelets/src/main/resources/kamelets/databricks-source.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/databricks-source.kamelet.yaml new file mode 100644 index 000000000..29d05a263 --- /dev/null +++ b/library/camel-kamelets/src/main/resources/kamelets/databricks-source.kamelet.yaml @@ -0,0 +1,114 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: databricks-source + annotations: + camel.apache.org/kamelet.support.level: "Preview" + camel.apache.org/catalog.version: "4.17.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA0MCA0MCIgd2lkdGg9IjQwIiBoZWlnaHQ9IjQwIj4KICA8cGF0aCBmaWxsPSIjRkYzNjIxIiBkPSJNMjAgMEw0MCA4LjI3VjEzLjFMMjAgMjAuNUwwIDEzLjFWOC4yN0wyMCAwWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik0wIDEzLjFMMjAgMjAuNVYyNi45TDAgMTkuNVYxMy4xWiIvPgogIDxwYXRoIGZpbGw9IiNGRjM2MjEiIGQ9Ik00MCAxMy4xTDIwIDIwLjVWMjYuOUw0MCAxOS41VjEzLjFaIi8+CiAgPHBhdGggZmlsbD0iI0ZGMzYyMSIgZD0iTTAgMTkuNUwyMC [...] + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "SQL" + camel.apache.org/kamelet.namespace: "Database" + labels: + camel.apache.org/kamelet.type: "source" +spec: + definition: + title: "Databricks Source" + description: Query data from a Databricks Database. For Unity Catalog workspaces, specify catalog and schema parameters. + required: + - serverHostname + - httpPath + - accessToken + - query + type: object + properties: + serverHostname: + title: Server Hostname + description: The Databricks server hostname. + type: string + example: adb-1234567890123456.7.azuredatabricks.net + serverPort: + title: Server Port + description: The server port for the Databricks data source. + type: string + default: 443 + httpPath: + title: HTTP Path + description: The HTTP path to the Databricks SQL Warehouse or cluster. + type: string + example: /sql/1.0/warehouses/abc123def456 + accessToken: + title: Access Token + description: The personal access token to access Databricks. + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + query: + title: Query + description: The query to execute against the Databricks Database. + type: string + example: 'SELECT * FROM accounts' + extraOptions: + title: Extra Options + description: Additional JDBC connection options (e.g., ConnCatalog=main;ConnSchema=default). + type: string + default: '' + consumedQuery: + title: Consumed Query + description: A query to run on a tuple consumed. + type: string + example: 'DELETE FROM accounts where user_id = :#user_id' + delay: + title: Delay + description: The number of milliseconds before the next poll from the Databricks database. + type: integer + default: 500 + types: + out: + mediaType: application/json + dependencies: + - "camel:jackson" + - "camel:kamelet" + - "camel:sql" + - "mvn:com.databricks:databricks-jdbc:3.0.7" + - "mvn:org.apache.commons:commons-dbcp2:2.14.0" + template: + beans: + - name: local-sql-databricks-source + type: "#class:org.apache.camel.component.sql.SqlComponent" + properties: + autowiredEnabled: "false" + - name: dsBean + type: "#class:org.apache.commons.dbcp2.BasicDataSource" + properties: + password: '{{accessToken}}' + url: 'jdbc:databricks://{{serverHostname}}:{{serverPort}};httpPath={{httpPath}};AuthMech=3;transportMode=http;ssl=1;UID=token;PWD={{accessToken}}{{extraOptions}}' + driverClassName: 'com.databricks.client.jdbc.Driver' + from: + uri: "{{local-sql-databricks-source}}:{{query}}" + parameters: + dataSource: "#bean:{{dsBean}}" + onConsume: "{{?consumedQuery}}" + delay: "{{delay}}" + steps: + - marshal: + json: + library: Jackson + - to: "kamelet:sink"
