This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
The following commit(s) were added to refs/heads/main by this push:
new bb72a97a2 Fix #3360: new debezium-postgresql-source Kamelet, the
changes of PostgreSQL tables as they happen (#3363)
bb72a97a2 is described below
commit bb72a97a25e195562ea6ffea35e67b51027ad7ad
Author: Claus Ibsen <[email protected]>
AuthorDate: Sun Nov 22 21:03:26 2026 +0100
Fix #3360: new debezium-postgresql-source Kamelet, the changes of
PostgreSQL tables as they happen (#3363)
The catalog had no change data capture source. debezium-postgresql-source,
on the
Camel debezium-postgres component, sends one message per row change: the
row after
the change (before it, for a delete) as JSON text, with the
debezium-operation,
debezium-table and debezium-key headers. Tombstones and other events
without a row
are skipped, and the Kafka Connect objects in the CamelDebezium* headers
are removed.
It takes a curated set of options: the connection, the tables, the snapshot
mode,
the replication slot, the publication and the decoding plugin (pgoutput by
default),
and where the position is saved: a file, or a Kafka topic. Any other
connector
option can be set as camel.component.debezium-postgres.<option>.
Citrus tests (DatabaseIT) on PostgreSQL 17 with wal_level=logical: the
snapshot,
an insert, an update and a delete arrive with the right operation, table,
key and
body; a table outside tableIncludeList is not sent; and a restart with the
same
position file, or with the Kafka store, sends only the change made while it
was
stopped.
Claude-Session: https://claude.ai/code/session_016Y4dDmBDsSWV117kccuNf9
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../camel-kamelets-catalog-sbom.json | 24 +-
docs/modules/ROOT/nav.adoc | 1 +
kamelets/debezium-postgresql-source.kamelet.yaml | 270 ++++++++++++++++++++
.../main/resources/META-INF/kamelets-index.json | 10 +
.../debezium-postgresql-source.kamelet.yaml | 270 ++++++++++++++++++++
.../src/test/java/DatabaseIT.java | 5 +
...um-postgresql-source-kafka-route.citrus.it.yaml | 148 +++++++++++
.../debezium-postgresql-source-kafka-route.yaml | 42 ++++
...debezium-postgresql-source-route.citrus.it.yaml | 277 +++++++++++++++++++++
.../debezium/debezium-postgresql-source-route.yaml | 54 ++++
.../src/test/resources/database/debezium/init.sql | 3 +
.../src/test/resources/database/sql.groovy | 4 +
12 files changed, 1107 insertions(+), 1 deletion(-)
diff --git a/camel-kamelets-sbom/camel-kamelets-catalog-sbom.json
b/camel-kamelets-sbom/camel-kamelets-catalog-sbom.json
index 4940d33c2..146e250ab 100644
--- a/camel-kamelets-sbom/camel-kamelets-catalog-sbom.json
+++ b/camel-kamelets-sbom/camel-kamelets-catalog-sbom.json
@@ -341,6 +341,11 @@
"type" : "library",
"name" : "databricks-source",
"version" : "4.24.0-SNAPSHOT"
+ }, {
+ "bom-ref" : "kamelet:debezium-postgresql-source",
+ "type" : "library",
+ "name" : "debezium-postgresql-source",
+ "version" : "4.24.0-SNAPSHOT"
}, {
"bom-ref" : "kamelet:delay-action",
"type" : "library",
@@ -1563,6 +1568,17 @@
"name" : "camel.apache.org/versioned-by",
"value" : "runtime"
} ]
+ }, {
+ "bom-ref" :
"pkg:maven/org.apache.camel/[email protected]",
+ "type" : "library",
+ "group" : "org.apache.camel",
+ "name" : "camel-debezium-postgres",
+ "version" : "4.24.0-SNAPSHOT",
+ "purl" :
"pkg:maven/org.apache.camel/[email protected]",
+ "properties" : [ {
+ "name" : "camel.apache.org/versioned-by",
+ "value" : "runtime"
+ } ]
}, {
"bom-ref" : "pkg:maven/org.apache.camel/[email protected]",
"type" : "library",
@@ -2463,7 +2479,7 @@
} ],
"dependencies" : [ {
"ref" : "camel-kamelets-catalog",
- "dependsOn" : [ "kamelet:avro-deserialize-action",
"kamelet:avro-serialize-action", "kamelet:aws-bedrock-agent-runtime-sink",
"kamelet:aws-bedrock-agent-sink", "kamelet:aws-bedrock-text-sink",
"kamelet:aws-cloudtrail-source", "kamelet:aws-cloudwatch-sink",
"kamelet:aws-comprehend-sink", "kamelet:aws-ddb-sink",
"kamelet:aws-ddb-streams-source", "kamelet:aws-ec2-sink",
"kamelet:aws-eventbridge-sink", "kamelet:aws-eventbridge-source",
"kamelet:aws-kinesis-firehose-sink", "kamelet:aws-ki [...]
+ "dependsOn" : [ "kamelet:avro-deserialize-action",
"kamelet:avro-serialize-action", "kamelet:aws-bedrock-agent-runtime-sink",
"kamelet:aws-bedrock-agent-sink", "kamelet:aws-bedrock-text-sink",
"kamelet:aws-cloudtrail-source", "kamelet:aws-cloudwatch-sink",
"kamelet:aws-comprehend-sink", "kamelet:aws-ddb-sink",
"kamelet:aws-ddb-streams-source", "kamelet:aws-ec2-sink",
"kamelet:aws-eventbridge-sink", "kamelet:aws-eventbridge-source",
"kamelet:aws-kinesis-firehose-sink", "kamelet:aws-ki [...]
}, {
"ref" : "kamelet:avro-deserialize-action",
"dependsOn" : [
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]" ]
@@ -2662,6 +2678,9 @@
}, {
"ref" : "kamelet:databricks-source",
"dependsOn" : [ "pkg:maven/com.databricks/[email protected]",
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.commons/[email protected]" ]
+ }, {
+ "ref" : "kamelet:debezium-postgresql-source",
+ "dependsOn" : [
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]",
"pkg:maven/org.apache.camel/[email protected]" ]
}, {
"ref" : "kamelet:delay-action",
"dependsOn" : [ "pkg:maven/org.apache.camel/[email protected]"
]
@@ -3274,6 +3293,9 @@
}, {
"ref" : "pkg:maven/org.apache.camel/[email protected]",
"dependsOn" : [ ]
+ }, {
+ "ref" :
"pkg:maven/org.apache.camel/[email protected]",
+ "dependsOn" : [ ]
}, {
"ref" : "pkg:maven/org.apache.camel/[email protected]",
"dependsOn" : [ ]
diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc
index 9160f276f..e28f7f584 100644
--- a/docs/modules/ROOT/nav.adoc
+++ b/docs/modules/ROOT/nav.adoc
@@ -65,6 +65,7 @@
* xref:data-type-action.adoc[]
* xref:databricks-sink.adoc[]
* xref:databricks-source.adoc[]
+* xref:debezium-postgresql-source.adoc[]
* xref:delay-action.adoc[]
* xref:dns-dig-action.adoc[]
* xref:dns-ip-action.adoc[]
diff --git a/kamelets/debezium-postgresql-source.kamelet.yaml
b/kamelets/debezium-postgresql-source.kamelet.yaml
new file mode 100644
index 000000000..b38a8d578
--- /dev/null
+++ b/kamelets/debezium-postgresql-source.kamelet.yaml
@@ -0,0 +1,270 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: debezium-postgresql-source
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.24.0-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+UG9zdGdyZVNRTCBpY29uPC90aXRsZT48cGF0aCBkPSJNMTcuMTI4IDBhMTAuMTM0IDEwLjEzNCAwIDAgMC0yLjc1NS40MDNsLS4wNjMuMDJBMTAuOTIyIDEwLjkyMiAwIDAgMCAxMi42LjI1OEMxMS40MjIuMjM4IDEwLjQxLjUyNCA5LjU5NCAxIDguNzkuNzIxIDcuMTIyLjI0IDUuMzY0LjMzNiA0LjE0LjQwMyAyLjgwNC43NzUgMS44MTQgMS44Mi44MjcgMi44NjUuMzA1IDQuNDgyLjQxNSA2LjY4MmMuMDMuNjA3LjIwMyAxLjU5Ny
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "Debezium"
+ camel.apache.org/kamelet.namespace: "Database"
+ labels:
+ camel.apache.org/kamelet.type: "source"
+ camel.apache.org/kamelet.verified: "true"
+spec:
+ definition:
+ title: "Debezium PostgreSQL Source"
+ description: |-
+ Sends a message for each row inserted, updated or deleted in the tables
of a PostgreSQL database, as it happens (change data capture with Debezium).
+
+ Use it to pass the changes of a database on to another system as they
happen, for example to keep a search index or a cache up to date, or to copy
new orders to a queue, without polling the tables. To read rows with a query
again and again, use `postgresql-source`.
+
+ Output: the body is JSON text in bytes, an object with the column names
as keys, such as `{"id":1,"name":"Tea","price":4.50}`: the row after the
change, and for a delete the row before it. The columns can come in any order.
A `NUMERIC` column is a JSON number with its decimals, such as `4.50`, and a
`DATE` column the number of days since 1970-01-01, such as `20454` for
2026-01-01. The `debezium-operation` header says what happened: `c` (insert),
`u` (update), `d` (delete) or `r` (a [...]
+
+ Good to know: the server must run with `wal_level=logical`, and the user
needs the `REPLICATION` attribute and the right to read the tables. On the
first start, the Kamelet creates a replication slot named `slotName` and a
publication named `publicationName` in the database, and by default sends every
row of the tables first (`snapshotMode`). It saves how far it got in the file
`offsetStorageFileName`, or in a Kafka topic, when it stops and once a minute
while it runs, and after a [...]
+ required:
+ - serverName
+ - username
+ - password
+ - databaseName
+ - topicPrefix
+ type: object
+ properties:
+ serverName:
+ title: Server Name
+ description: The host name or IP address of the PostgreSQL server,
without the port. It must be the primary server, not a read replica.
+ type: string
+ example: db.example.com
+ serverPort:
+ title: Server Port
+ description: The port the PostgreSQL server listens on. PostgreSQL
uses 5432 unless it was set up otherwise.
+ type: integer
+ default: 5432
+ username:
+ title: Username
+ description: The database user to log in as. It needs the
`REPLICATION` attribute, the right to read the tables and, unless the
publication already exists, the right to create it (to create a publication for
all tables, the user must be a superuser).
+ type: string
+ x-descriptors:
+ - urn:camel:group:credentials
+ password:
+ title: Password
+ description: The password of the database user `username`.
+ type: string
+ format: password
+ x-descriptors:
+ - urn:camel:group:credentials
+ databaseName:
+ title: Database Name
+ description: The name of the database whose changes to send.
+ type: string
+ example: shop
+ topicPrefix:
+ title: Logical Name
+ description: A name for this source of changes, made of letters,
digits, `.`, `_` and `-`. The saved position is stored under this name, so keep
it the same across restarts, and give each Kamelet that reads another database
its own name.
+ type: string
+ pattern: "^[a-zA-Z0-9._-]+$"
+ example: shop
+ tableIncludeList:
+ title: Tables
+ description: The tables whose changes to send, as regular expressions
that match `schema.table`, separated by commas, such as
`public.orders,public.customers`. When it is not set, the changes of all tables
are sent.
+ type: string
+ example: public.orders,public.customers
+ snapshotMode:
+ title: Snapshot Mode
+ description: "Whether to send the rows that are already in the tables
before the changes. `initial` (the default): on the first start, every row of
the tables is sent with the operation `r`, then the changes; after a restart
with a saved position, only the changes since. `no_data`: only the changes made
from the first start on. `always`: every row on each start, then the changes.
`initial_only`: every row on the first start, and no changes. `when_needed`: as
`initial`, and every [...]
+ type: string
+ enum: ["initial", "no_data", "always", "initial_only", "when_needed"]
+ default: initial
+ slotName:
+ title: Replication Slot
+ description: The name of the PostgreSQL replication slot that keeps
the changes until the Kamelet has read them, made of lowercase letters, digits
and `_`. It is created when it does not exist. Each Kamelet that reads the same
database needs its own slot.
+ type: string
+ pattern: "^[a-z0-9_]+$"
+ default: debezium
+ publicationName:
+ title: Publication
+ description: The name of the PostgreSQL publication that says which
tables are logged for the Kamelet, made of letters, digits and `_`. See
`publicationAutocreateMode` for whether it is created.
+ type: string
+ pattern: "^[a-zA-Z0-9_]+$"
+ default: dbz_publication
+ publicationAutocreateMode:
+ title: Create Publication
+ description: "Whether the Kamelet creates the publication when it does
not exist. `all_tables` (the default): for all tables, which needs a superuser.
`filtered`: for the tables of `tableIncludeList` only, which needs only to own
them. `disabled`: never; the Kamelet fails to start when the publication does
not exist. `no_tables`: for no table, so that the tables can be added to it
later. An existing publication is used as it is."
+ type: string
+ enum: ["all_tables", "filtered", "disabled", "no_tables"]
+ default: all_tables
+ pluginName:
+ title: Decoding Plugin
+ description: "The PostgreSQL plugin that decodes the changes.
`pgoutput` is built into PostgreSQL 10 and newer. `decoderbufs` must be
installed on the server."
+ type: string
+ enum: ["pgoutput", "decoderbufs"]
+ default: pgoutput
+ offsetStorageFileName:
+ title: Position File
+ description: The file where the Kamelet saves how far it got, such as
`/data/offsets.dat`, read again after a restart. It is created when it does not
exist, but its folder must exist, or nothing is saved. A relative name is in
the folder the integration runs in. When it is lost, the changes are sent again
as on the first start. Not used when `offsetStorage` is the Kafka store.
+ type: string
+ default: debezium-postgresql-offsets.dat
+ example: /data/offsets.dat
+ offsetStorage:
+ title: Position Store
+ description: "Where the Kamelet saves how far it got.
`org.apache.kafka.connect.storage.FileOffsetBackingStore` (the default): in the
file `offsetStorageFileName`.
`org.apache.kafka.connect.storage.KafkaOffsetBackingStore`: in the Kafka topic
`offsetStorageTopic`, on the Kafka of `offsetStorageBootstrapServers`, for an
integration that has no lasting disk."
+ type: string
+ enum: ["org.apache.kafka.connect.storage.FileOffsetBackingStore",
"org.apache.kafka.connect.storage.KafkaOffsetBackingStore"]
+ default: org.apache.kafka.connect.storage.FileOffsetBackingStore
+ offsetStorageBootstrapServers:
+ title: Position Kafka Brokers
+ description: The Kafka brokers of the Kafka store, as `host:port`
separated by commas. Needed with the Kafka store. The connection is made
without TLS and without a login.
+ type: string
+ example: my-cluster-kafka-bootstrap:9092
+ offsetStorageTopic:
+ title: Position Topic
+ description: The Kafka topic where the Kafka store saves how far the
Kamelet got. It is created when it does not exist. Needed with the Kafka store.
+ type: string
+ example: shop-offsets
+ offsetStoragePartitions:
+ title: Position Topic Partitions
+ description: The number of partitions of the topic
`offsetStorageTopic` when the Kamelet creates it. Needed with the Kafka store.
+ type: integer
+ example: 1
+ offsetStorageReplicationFactor:
+ title: Position Topic Replicas
+ description: The number of copies of the topic `offsetStorageTopic` on
the Kafka brokers when the Kamelet creates it. Needed with the Kafka store.
+ type: integer
+ example: 3
+ dataTypes:
+ out:
+ default: json
+ types:
+ json:
+ format: "application-json"
+ description: "The row after the change, or before it for a delete,
as JSON text in bytes."
+ mediaType: application/json
+ headers:
+ debezium-operation:
+ title: Operation
+ description: "What happened to the row: `c` (insert), `u` (update),
`d` (delete) or `r` (read by the snapshot)."
+ type: string
+ debezium-table:
+ title: Table
+ description: The table of the row, as `schema.table`, such as
`public.orders`.
+ type: string
+ debezium-key:
+ title: Key
+ description: The primary key of the row, as JSON text such as
`{"id":1}`.
+ type: string
+ Content-Type:
+ title: Content Type
+ description: Always `application/json`, as the body is JSON text.
+ type: string
+ dependencies:
+ - "camel:jackson3"
+ - "camel:kamelet"
+ - "camel:debezium-postgres"
+ template:
+ from:
+ uri: debezium-postgres
+ parameters:
+ name: "{{topicPrefix}}"
+ databaseHostname: "{{serverName}}"
+ databasePort: "{{serverPort}}"
+ databaseUser: "{{username}}"
+ databasePassword: "RAW({{password}})"
+ databaseDbname: "{{databaseName}}"
+ topicPrefix: "{{topicPrefix}}"
+ tableIncludeList: "{{?tableIncludeList}}"
+ snapshotMode: "{{snapshotMode}}"
+ slotName: "{{slotName}}"
+ publicationName: "{{publicationName}}"
+ publicationAutocreateMode: "{{publicationAutocreateMode}}"
+ pluginName: "{{pluginName}}"
+ offsetStorage: "{{offsetStorage}}"
+ offsetStorageFileName: "{{offsetStorageFileName}}"
+ offsetStorageTopic: "{{?offsetStorageTopic}}"
+ offsetStoragePartitions: "{{?offsetStoragePartitions}}"
+ offsetStorageReplicationFactor: "{{?offsetStorageReplicationFactor}}"
+ additionalProperties.bootstrap.servers:
"{{?offsetStorageBootstrapServers}}"
+ steps:
+ # only the changes of rows: not the tombstones, heartbeats or logical
decoding messages, which have no row
+ - filter:
+ expression:
+ simple:
+ expression: "${header.CamelDebeziumOperation} in 'c,u,d,r'"
+ steps:
+ - setHeader:
+ name: debezium-operation
+ expression:
+ simple:
+ expression: "${header.CamelDebeziumOperation}"
+ - setHeader:
+ name: debezium-table
+ expression:
+ simple:
+ expression:
"${header.CamelDebeziumSourceMetadata[schema]}.${header.CamelDebeziumSourceMetadata[table]}"
+ # the key as JSON text
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${header.CamelDebeziumKey} != null"
+ steps:
+ - setProperty:
+ name: debezium-row
+ expression:
+ simple:
+ expression: "${body}"
+ - setBody:
+ expression:
+ simple:
+ expression: "${header.CamelDebeziumKey}"
+ - convertBodyTo:
+ type: java.util.Map
+ - marshal:
+ json: {}
+ - setHeader:
+ name: debezium-key
+ expression:
+ simple:
+ expression: "${bodyAs(String)}"
+ - setBody:
+ expression:
+ simple:
+ expression: "${exchangeProperty.debezium-row}"
+ - removeProperty:
+ name: debezium-row
+ # a delete has no row after it: send the row before it
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${header.CamelDebeziumOperation} == 'd'"
+ steps:
+ - setBody:
+ expression:
+ simple:
+ expression: "${header.CamelDebeziumBefore}"
+ - convertBodyTo:
+ type: java.util.Map
+ - marshal:
+ json: {}
+ # the Kafka Connect objects of the component, which other
systems cannot read
+ - removeHeaders:
+ pattern: CamelDebezium*
+ - to:
+ uri: kamelet:sink
diff --git
a/library/camel-kamelets/src/main/resources/META-INF/kamelets-index.json
b/library/camel-kamelets/src/main/resources/META-INF/kamelets-index.json
index fc12a251f..abc3a712c 100644
--- a/library/camel-kamelets/src/main/resources/META-INF/kamelets-index.json
+++ b/library/camel-kamelets/src/main/resources/META-INF/kamelets-index.json
@@ -659,6 +659,16 @@
"group" : "SQL",
"namespace" : "Database",
"provider" : "Apache Software Foundation"
+ }, {
+ "name" : "debezium-postgresql-source",
+ "type" : "source",
+ "supportLevel" : "Preview",
+ "title" : "Debezium PostgreSQL Source",
+ "summary" : "Sends a message for each row inserted, updated or deleted in
the tables of a PostgreSQL database, as it happens (change data capture with
Debezium).",
+ "description" : "Sends a message for each row inserted, updated or deleted
in the tables of a PostgreSQL database, as it happens (change data capture with
Debezium).\n\nUse it to pass the changes of a database on to another system as
they happen, for example to keep a search index or a cache up to date, or to
copy new orders to a queue, without polling the tables. To read rows with a
query again and again, use `postgresql-source`.\n\nOutput: the body is JSON
text in bytes, an object [...]
+ "group" : "Debezium",
+ "namespace" : "Database",
+ "provider" : "Apache Software Foundation"
}, {
"name" : "delay-action",
"type" : "action",
diff --git
a/library/camel-kamelets/src/main/resources/kamelets/debezium-postgresql-source.kamelet.yaml
b/library/camel-kamelets/src/main/resources/kamelets/debezium-postgresql-source.kamelet.yaml
new file mode 100644
index 000000000..b38a8d578
--- /dev/null
+++
b/library/camel-kamelets/src/main/resources/kamelets/debezium-postgresql-source.kamelet.yaml
@@ -0,0 +1,270 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+apiVersion: camel.apache.org/v1
+kind: Kamelet
+metadata:
+ name: debezium-postgresql-source
+ annotations:
+ camel.apache.org/kamelet.support.level: "Preview"
+ camel.apache.org/catalog.version: "4.24.0-SNAPSHOT"
+ camel.apache.org/kamelet.icon:
"data:image/svg+xml;base64,PHN2ZyByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+UG9zdGdyZVNRTCBpY29uPC90aXRsZT48cGF0aCBkPSJNMTcuMTI4IDBhMTAuMTM0IDEwLjEzNCAwIDAgMC0yLjc1NS40MDNsLS4wNjMuMDJBMTAuOTIyIDEwLjkyMiAwIDAgMCAxMi42LjI1OEMxMS40MjIuMjM4IDEwLjQxLjUyNCA5LjU5NCAxIDguNzkuNzIxIDcuMTIyLjI0IDUuMzY0LjMzNiA0LjE0LjQwMyAyLjgwNC43NzUgMS44MTQgMS44Mi44MjcgMi44NjUuMzA1IDQuNDgyLjQxNSA2LjY4MmMuMDMuNjA3LjIwMyAxLjU5Ny
[...]
+ camel.apache.org/provider: "Apache Software Foundation"
+ camel.apache.org/kamelet.group: "Debezium"
+ camel.apache.org/kamelet.namespace: "Database"
+ labels:
+ camel.apache.org/kamelet.type: "source"
+ camel.apache.org/kamelet.verified: "true"
+spec:
+ definition:
+ title: "Debezium PostgreSQL Source"
+ description: |-
+ Sends a message for each row inserted, updated or deleted in the tables
of a PostgreSQL database, as it happens (change data capture with Debezium).
+
+ Use it to pass the changes of a database on to another system as they
happen, for example to keep a search index or a cache up to date, or to copy
new orders to a queue, without polling the tables. To read rows with a query
again and again, use `postgresql-source`.
+
+ Output: the body is JSON text in bytes, an object with the column names
as keys, such as `{"id":1,"name":"Tea","price":4.50}`: the row after the
change, and for a delete the row before it. The columns can come in any order.
A `NUMERIC` column is a JSON number with its decimals, such as `4.50`, and a
`DATE` column the number of days since 1970-01-01, such as `20454` for
2026-01-01. The `debezium-operation` header says what happened: `c` (insert),
`u` (update), `d` (delete) or `r` (a [...]
+
+ Good to know: the server must run with `wal_level=logical`, and the user
needs the `REPLICATION` attribute and the right to read the tables. On the
first start, the Kamelet creates a replication slot named `slotName` and a
publication named `publicationName` in the database, and by default sends every
row of the tables first (`snapshotMode`). It saves how far it got in the file
`offsetStorageFileName`, or in a Kafka topic, when it stops and once a minute
while it runs, and after a [...]
+ required:
+ - serverName
+ - username
+ - password
+ - databaseName
+ - topicPrefix
+ type: object
+ properties:
+ serverName:
+ title: Server Name
+ description: The host name or IP address of the PostgreSQL server,
without the port. It must be the primary server, not a read replica.
+ type: string
+ example: db.example.com
+ serverPort:
+ title: Server Port
+ description: The port the PostgreSQL server listens on. PostgreSQL
uses 5432 unless it was set up otherwise.
+ type: integer
+ default: 5432
+ username:
+ title: Username
+ description: The database user to log in as. It needs the
`REPLICATION` attribute, the right to read the tables and, unless the
publication already exists, the right to create it (to create a publication for
all tables, the user must be a superuser).
+ type: string
+ x-descriptors:
+ - urn:camel:group:credentials
+ password:
+ title: Password
+ description: The password of the database user `username`.
+ type: string
+ format: password
+ x-descriptors:
+ - urn:camel:group:credentials
+ databaseName:
+ title: Database Name
+ description: The name of the database whose changes to send.
+ type: string
+ example: shop
+ topicPrefix:
+ title: Logical Name
+ description: A name for this source of changes, made of letters,
digits, `.`, `_` and `-`. The saved position is stored under this name, so keep
it the same across restarts, and give each Kamelet that reads another database
its own name.
+ type: string
+ pattern: "^[a-zA-Z0-9._-]+$"
+ example: shop
+ tableIncludeList:
+ title: Tables
+ description: The tables whose changes to send, as regular expressions
that match `schema.table`, separated by commas, such as
`public.orders,public.customers`. When it is not set, the changes of all tables
are sent.
+ type: string
+ example: public.orders,public.customers
+ snapshotMode:
+ title: Snapshot Mode
+ description: "Whether to send the rows that are already in the tables
before the changes. `initial` (the default): on the first start, every row of
the tables is sent with the operation `r`, then the changes; after a restart
with a saved position, only the changes since. `no_data`: only the changes made
from the first start on. `always`: every row on each start, then the changes.
`initial_only`: every row on the first start, and no changes. `when_needed`: as
`initial`, and every [...]
+ type: string
+ enum: ["initial", "no_data", "always", "initial_only", "when_needed"]
+ default: initial
+ slotName:
+ title: Replication Slot
+ description: The name of the PostgreSQL replication slot that keeps
the changes until the Kamelet has read them, made of lowercase letters, digits
and `_`. It is created when it does not exist. Each Kamelet that reads the same
database needs its own slot.
+ type: string
+ pattern: "^[a-z0-9_]+$"
+ default: debezium
+ publicationName:
+ title: Publication
+ description: The name of the PostgreSQL publication that says which
tables are logged for the Kamelet, made of letters, digits and `_`. See
`publicationAutocreateMode` for whether it is created.
+ type: string
+ pattern: "^[a-zA-Z0-9_]+$"
+ default: dbz_publication
+ publicationAutocreateMode:
+ title: Create Publication
+ description: "Whether the Kamelet creates the publication when it does
not exist. `all_tables` (the default): for all tables, which needs a superuser.
`filtered`: for the tables of `tableIncludeList` only, which needs only to own
them. `disabled`: never; the Kamelet fails to start when the publication does
not exist. `no_tables`: for no table, so that the tables can be added to it
later. An existing publication is used as it is."
+ type: string
+ enum: ["all_tables", "filtered", "disabled", "no_tables"]
+ default: all_tables
+ pluginName:
+ title: Decoding Plugin
+ description: "The PostgreSQL plugin that decodes the changes.
`pgoutput` is built into PostgreSQL 10 and newer. `decoderbufs` must be
installed on the server."
+ type: string
+ enum: ["pgoutput", "decoderbufs"]
+ default: pgoutput
+ offsetStorageFileName:
+ title: Position File
+ description: The file where the Kamelet saves how far it got, such as
`/data/offsets.dat`, read again after a restart. It is created when it does not
exist, but its folder must exist, or nothing is saved. A relative name is in
the folder the integration runs in. When it is lost, the changes are sent again
as on the first start. Not used when `offsetStorage` is the Kafka store.
+ type: string
+ default: debezium-postgresql-offsets.dat
+ example: /data/offsets.dat
+ offsetStorage:
+ title: Position Store
+ description: "Where the Kamelet saves how far it got.
`org.apache.kafka.connect.storage.FileOffsetBackingStore` (the default): in the
file `offsetStorageFileName`.
`org.apache.kafka.connect.storage.KafkaOffsetBackingStore`: in the Kafka topic
`offsetStorageTopic`, on the Kafka of `offsetStorageBootstrapServers`, for an
integration that has no lasting disk."
+ type: string
+ enum: ["org.apache.kafka.connect.storage.FileOffsetBackingStore",
"org.apache.kafka.connect.storage.KafkaOffsetBackingStore"]
+ default: org.apache.kafka.connect.storage.FileOffsetBackingStore
+ offsetStorageBootstrapServers:
+ title: Position Kafka Brokers
+ description: The Kafka brokers of the Kafka store, as `host:port`
separated by commas. Needed with the Kafka store. The connection is made
without TLS and without a login.
+ type: string
+ example: my-cluster-kafka-bootstrap:9092
+ offsetStorageTopic:
+ title: Position Topic
+ description: The Kafka topic where the Kafka store saves how far the
Kamelet got. It is created when it does not exist. Needed with the Kafka store.
+ type: string
+ example: shop-offsets
+ offsetStoragePartitions:
+ title: Position Topic Partitions
+ description: The number of partitions of the topic
`offsetStorageTopic` when the Kamelet creates it. Needed with the Kafka store.
+ type: integer
+ example: 1
+ offsetStorageReplicationFactor:
+ title: Position Topic Replicas
+ description: The number of copies of the topic `offsetStorageTopic` on
the Kafka brokers when the Kamelet creates it. Needed with the Kafka store.
+ type: integer
+ example: 3
+ dataTypes:
+ out:
+ default: json
+ types:
+ json:
+ format: "application-json"
+ description: "The row after the change, or before it for a delete,
as JSON text in bytes."
+ mediaType: application/json
+ headers:
+ debezium-operation:
+ title: Operation
+ description: "What happened to the row: `c` (insert), `u` (update),
`d` (delete) or `r` (read by the snapshot)."
+ type: string
+ debezium-table:
+ title: Table
+ description: The table of the row, as `schema.table`, such as
`public.orders`.
+ type: string
+ debezium-key:
+ title: Key
+ description: The primary key of the row, as JSON text such as
`{"id":1}`.
+ type: string
+ Content-Type:
+ title: Content Type
+ description: Always `application/json`, as the body is JSON text.
+ type: string
+ dependencies:
+ - "camel:jackson3"
+ - "camel:kamelet"
+ - "camel:debezium-postgres"
+ template:
+ from:
+ uri: debezium-postgres
+ parameters:
+ name: "{{topicPrefix}}"
+ databaseHostname: "{{serverName}}"
+ databasePort: "{{serverPort}}"
+ databaseUser: "{{username}}"
+ databasePassword: "RAW({{password}})"
+ databaseDbname: "{{databaseName}}"
+ topicPrefix: "{{topicPrefix}}"
+ tableIncludeList: "{{?tableIncludeList}}"
+ snapshotMode: "{{snapshotMode}}"
+ slotName: "{{slotName}}"
+ publicationName: "{{publicationName}}"
+ publicationAutocreateMode: "{{publicationAutocreateMode}}"
+ pluginName: "{{pluginName}}"
+ offsetStorage: "{{offsetStorage}}"
+ offsetStorageFileName: "{{offsetStorageFileName}}"
+ offsetStorageTopic: "{{?offsetStorageTopic}}"
+ offsetStoragePartitions: "{{?offsetStoragePartitions}}"
+ offsetStorageReplicationFactor: "{{?offsetStorageReplicationFactor}}"
+ additionalProperties.bootstrap.servers:
"{{?offsetStorageBootstrapServers}}"
+ steps:
+ # only the changes of rows: not the tombstones, heartbeats or logical
decoding messages, which have no row
+ - filter:
+ expression:
+ simple:
+ expression: "${header.CamelDebeziumOperation} in 'c,u,d,r'"
+ steps:
+ - setHeader:
+ name: debezium-operation
+ expression:
+ simple:
+ expression: "${header.CamelDebeziumOperation}"
+ - setHeader:
+ name: debezium-table
+ expression:
+ simple:
+ expression:
"${header.CamelDebeziumSourceMetadata[schema]}.${header.CamelDebeziumSourceMetadata[table]}"
+ # the key as JSON text
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${header.CamelDebeziumKey} != null"
+ steps:
+ - setProperty:
+ name: debezium-row
+ expression:
+ simple:
+ expression: "${body}"
+ - setBody:
+ expression:
+ simple:
+ expression: "${header.CamelDebeziumKey}"
+ - convertBodyTo:
+ type: java.util.Map
+ - marshal:
+ json: {}
+ - setHeader:
+ name: debezium-key
+ expression:
+ simple:
+ expression: "${bodyAs(String)}"
+ - setBody:
+ expression:
+ simple:
+ expression: "${exchangeProperty.debezium-row}"
+ - removeProperty:
+ name: debezium-row
+ # a delete has no row after it: send the row before it
+ - choice:
+ when:
+ - expression:
+ simple:
+ expression: "${header.CamelDebeziumOperation} == 'd'"
+ steps:
+ - setBody:
+ expression:
+ simple:
+ expression: "${header.CamelDebeziumBefore}"
+ - convertBodyTo:
+ type: java.util.Map
+ - marshal:
+ json: {}
+ # the Kafka Connect objects of the component, which other
systems cannot read
+ - removeHeaders:
+ pattern: CamelDebezium*
+ - to:
+ uri: kamelet:sink
diff --git a/tests/camel-kamelets-itest/src/test/java/DatabaseIT.java
b/tests/camel-kamelets-itest/src/test/java/DatabaseIT.java
index 0c351ddb6..f5fd3197d 100644
--- a/tests/camel-kamelets-itest/src/test/java/DatabaseIT.java
+++ b/tests/camel-kamelets-itest/src/test/java/DatabaseIT.java
@@ -77,6 +77,11 @@ public class DatabaseIT {
return
CitrusTestFactorySupport.factory(TestLoader.YAML).packageScan("database.sqlserver");
}
+ @CitrusTestFactory
+ public Stream<DynamicTest> debezium() {
+ return
CitrusTestFactorySupport.factory(TestLoader.YAML).packageScan("database.debezium");
+ }
+
@CitrusTestFactory
public Stream<DynamicTest> oracle() {
return
CitrusTestFactorySupport.factory(TestLoader.YAML).packageScan("database.oracle");
diff --git
a/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-kafka-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-kafka-route.citrus.it.yaml
new file mode 100644
index 000000000..49c3143d9
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-kafka-route.citrus.it.yaml
@@ -0,0 +1,148 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: debezium-postgresql-source-kafka-route-test
+actions:
+ # Start PostgreSQL with logical decoding, and a table of one row
+ - testcontainers:
+ start:
+ container:
+ name: "postgres"
+ image: "postgres:17.12-alpine"
+ command: "postgres -c wal_level=logical"
+ autoRemove: true
+ startUpTimeout: 120
+ exposedPorts:
+ - 5432
+ env:
+ - name: "POSTGRES_DB"
+ value: "testdb"
+ - name: "POSTGRES_USER"
+ value: "test"
+ - name: "POSTGRES_PASSWORD"
+ value: "test"
+ volumeMounts:
+ - file: "database/debezium/init.sql"
+ mountPath: "/docker-entrypoint-initdb.d/init.sql"
+ # only the server started after the init scripts listens on TCP
+ waitFor:
+ logMessage: '.*listening on IPv4 address.*'
+
+ # A Kafka to save the position in
+ - testcontainers:
+ start:
+ kafka:
+ implementation: "apache"
+ version: "4.4.0"
+
+ # The first run sends the row already in the table (the snapshot), and saves
its position in Kafka when it stops
+ # the test stops this run itself, so Citrus must not stop it at the end
+ - camel:
+ cli:
+ run:
+ autoRemove: false
+ waitForRunningState: false
+ integration:
+ name: "debezium-kafka-first"
+ file:
"database/debezium/debezium-postgresql-source-kafka-route.yaml"
+ systemProperties:
+ properties:
+ - name: "db.serverName"
+ value: "${CITRUS_TESTCONTAINERS_POSTGRES_HOST}"
+ - name: "db.serverPort"
+ value: "${CITRUS_TESTCONTAINERS_POSTGRES_PORT}"
+ - name: "kafka.bootstrapServers"
+ value: "${CITRUS_TESTCONTAINERS_KAFKA_BOOTSTRAP_SERVERS}"
+ - name: "run.name"
+ value: "first"
+ - name: "http.sink.url"
+ value: "http://localhost:${http.server.port}"
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result"
+ headers:
+ - name: "X-Run"
+ value: 'first'
+ - name: "X-Operation"
+ value: 'r'
+ body:
+ data: '{"id":1,"name":"Tea","price":4.50,"added":20454,"note":"not
sent"}'
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+ - camel:
+ cli:
+ stop:
+ integration: "debezium-kafka-first"
+
+ # A row inserted while no integration runs
+ - createVariables:
+ variables:
+ - name: "db.container"
+ value: "postgres"
+ - name: "db.sql"
+ value: |-
+ INSERT INTO products VALUES (2, 'Coffee', 3.20, '2026-03-01', 'x');
+ - groovy:
+ script:
+ file: "database/sql.groovy"
+
+ # The second run reads the position from Kafka: it sends only the new row,
not the snapshot again
+ - camel:
+ cli:
+ run:
+ waitForRunningState: false
+ integration:
+ name: "debezium-kafka-second"
+ file:
"database/debezium/debezium-postgresql-source-kafka-route.yaml"
+ systemProperties:
+ properties:
+ - name: "db.serverName"
+ value: "${CITRUS_TESTCONTAINERS_POSTGRES_HOST}"
+ - name: "db.serverPort"
+ value: "${CITRUS_TESTCONTAINERS_POSTGRES_PORT}"
+ - name: "kafka.bootstrapServers"
+ value: "${CITRUS_TESTCONTAINERS_KAFKA_BOOTSTRAP_SERVERS}"
+ - name: "run.name"
+ value: "second"
+ - name: "http.sink.url"
+ value: "http://localhost:${http.server.port}"
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result"
+ headers:
+ - name: "X-Run"
+ value: 'second'
+ - name: "X-Operation"
+ value: 'c'
+ body:
+ data:
'{"id":2,"name":"Coffee","price":3.20,"added":20513,"note":"x"}'
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-kafka-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-kafka-route.yaml
new file mode 100644
index 000000000..c2e7b88e0
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-kafka-route.yaml
@@ -0,0 +1,42 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+- route:
+ from:
+ uri: "kamelet:debezium-postgresql-source"
+ parameters:
+ serverName: "{{db.serverName}}"
+ serverPort: "{{db.serverPort}}"
+ databaseName: "testdb"
+ username: "test"
+ password: "test"
+ topicPrefix: "shop"
+ tableIncludeList: "public.products"
+ # the position is saved in a Kafka topic, not in a file
+ offsetStorage:
"org.apache.kafka.connect.storage.KafkaOffsetBackingStore"
+ offsetStorageBootstrapServers: "{{kafka.bootstrapServers}}"
+ offsetStorageTopic: "shop-offsets"
+ offsetStoragePartitions: 1
+ offsetStorageReplicationFactor: 1
+ steps:
+ - setHeader:
+ name: X-Operation
+ simple: "${header.debezium-operation}"
+ - setHeader:
+ name: X-Run
+ simple: "{{run.name}}"
+ - to:
+ uri: "{{http.sink.url}}/result"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-route.citrus.it.yaml
b/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-route.citrus.it.yaml
new file mode 100644
index 000000000..a36df7e77
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-route.citrus.it.yaml
@@ -0,0 +1,277 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+name: debezium-postgresql-source-route-test
+variables:
+ # a new file for each run, so that no saved position of an earlier run is
used, in a folder that exists
+ - name: "offsetFile"
+ value: "citrus:concat(citrus:systemProperty('java.io.tmpdir'),
'/debezium-offsets-', citrus:randomUUID(), '.dat')"
+actions:
+ # Start PostgreSQL with logical decoding, and a table of one row
+ - testcontainers:
+ start:
+ container:
+ name: "postgres"
+ image: "postgres:17.12-alpine"
+ command: "postgres -c wal_level=logical"
+ autoRemove: true
+ startUpTimeout: 120
+ exposedPorts:
+ - 5432
+ env:
+ - name: "POSTGRES_DB"
+ value: "testdb"
+ - name: "POSTGRES_USER"
+ value: "test"
+ - name: "POSTGRES_PASSWORD"
+ value: "test"
+ volumeMounts:
+ - file: "database/debezium/init.sql"
+ mountPath: "/docker-entrypoint-initdb.d/init.sql"
+ # only the server started after the init scripts listens on TCP
+ waitFor:
+ logMessage: '.*listening on IPv4 address.*'
+
+ # The first run sends the row already in the table (the snapshot), then the
changes
+ # the test stops this run itself, so Citrus must not stop it at the end
+ - camel:
+ cli:
+ run:
+ autoRemove: false
+ waitForRunningState: false
+ integration:
+ name: "debezium-first"
+ file: "database/debezium/debezium-postgresql-source-route.yaml"
+ systemProperties:
+ properties:
+ - name: "db.serverName"
+ value: "${CITRUS_TESTCONTAINERS_POSTGRES_HOST}"
+ - name: "db.serverPort"
+ value: "${CITRUS_TESTCONTAINERS_POSTGRES_PORT}"
+ - name: "db.offsetFile"
+ value: "${offsetFile}"
+ - name: "run.name"
+ value: "first"
+ - name: "http.sink.url"
+ value: "http://localhost:${http.server.port}"
+ # any other option of the connector, for all its Kamelets
+ - name: "camel.component.debezium-postgres.columnExcludeList"
+ value: "public.products.note"
+
+ # The snapshot: the row as it is, as JSON text in bytes; the note column is
left out by columnExcludeList
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result"
+ headers:
+ - name: "X-Run"
+ value: 'first'
+ - name: "X-Body-Type"
+ value: '[B'
+ - name: "X-Content-Type"
+ value: 'ct=application/json'
+ - name: "X-Operation"
+ value: 'r'
+ - name: "X-Table"
+ value: 'public.products'
+ - name: "X-Key"
+ value: '{"id":1}'
+ - name: "X-Component-Headers"
+ value: 'op= key='
+ body:
+ data: '{"id":1,"name":"Tea","price":4.50,"added":20454}'
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # An insert, an update and a delete
+ - createVariables:
+ variables:
+ - name: "db.container"
+ value: "postgres"
+ - name: "db.sql"
+ value: |-
+ INSERT INTO products VALUES (2, 'Coffee', 3.20, '2026-03-01', 'not
sent');
+ UPDATE products SET price = 5.00 WHERE id = 1;
+ DELETE FROM products WHERE id = 2;
+ - groovy:
+ script:
+ file: "database/sql.groovy"
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result"
+ headers:
+ - name: "X-Run"
+ value: 'first'
+ - name: "X-Body-Type"
+ value: '[B'
+ - name: "X-Content-Type"
+ value: 'ct=application/json'
+ - name: "X-Operation"
+ value: 'c'
+ - name: "X-Table"
+ value: 'public.products'
+ - name: "X-Key"
+ value: '{"id":2}'
+ - name: "X-Component-Headers"
+ value: 'op= key='
+ body:
+ data: '{"id":2,"name":"Coffee","price":3.20,"added":20513}'
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result"
+ headers:
+ - name: "X-Run"
+ value: 'first'
+ - name: "X-Body-Type"
+ value: '[B'
+ - name: "X-Content-Type"
+ value: 'ct=application/json'
+ - name: "X-Operation"
+ value: 'u'
+ - name: "X-Table"
+ value: 'public.products'
+ - name: "X-Key"
+ value: '{"id":1}'
+ - name: "X-Component-Headers"
+ value: 'op= key='
+ body:
+ data: '{"id":1,"name":"Tea","price":5.00,"added":20454}'
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # A delete sends the row before it: without REPLICA IDENTITY FULL,
PostgreSQL logs only its key
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result"
+ headers:
+ - name: "X-Run"
+ value: 'first'
+ - name: "X-Body-Type"
+ value: '[B'
+ - name: "X-Content-Type"
+ value: 'ct=application/json'
+ - name: "X-Operation"
+ value: 'd'
+ - name: "X-Table"
+ value: 'public.products'
+ - name: "X-Key"
+ value: '{"id":2}'
+ - name: "X-Component-Headers"
+ value: 'op= key='
+ body:
+ data: '{"id":2,"name":null,"price":null,"added":null}'
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
+
+ # Stop, change a row while no integration runs, and start again with the
same position file
+ - camel:
+ cli:
+ stop:
+ integration: "debezium-first"
+ - createVariables:
+ variables:
+ - name: "db.container"
+ value: "postgres"
+ - name: "db.sql"
+ value: |-
+ INSERT INTO notes VALUES (1, 'not in tableIncludeList, so not
sent');
+ INSERT INTO products VALUES (3, 'Juice', 2.10, '2026-01-01', 'not
sent');
+ - groovy:
+ script:
+ file: "database/sql.groovy"
+ - camel:
+ cli:
+ run:
+ waitForRunningState: false
+ integration:
+ name: "debezium-second"
+ file: "database/debezium/debezium-postgresql-source-route.yaml"
+ systemProperties:
+ properties:
+ - name: "db.serverName"
+ value: "${CITRUS_TESTCONTAINERS_POSTGRES_HOST}"
+ - name: "db.serverPort"
+ value: "${CITRUS_TESTCONTAINERS_POSTGRES_PORT}"
+ - name: "db.offsetFile"
+ value: "${offsetFile}"
+ - name: "run.name"
+ value: "second"
+ - name: "http.sink.url"
+ value: "http://localhost:${http.server.port}"
+ # any other option of the connector, for all its Kamelets
+ - name: "camel.component.debezium-postgres.columnExcludeList"
+ value: "public.products.note"
+
+ # The second run sends only the change of products made while it was
stopped, not the snapshot or the changes sent before
+ - http:
+ server: "httpServer"
+ receiveRequest:
+ POST:
+ path: "/result"
+ headers:
+ - name: "X-Run"
+ value: 'second'
+ - name: "X-Body-Type"
+ value: '[B'
+ - name: "X-Content-Type"
+ value: 'ct=application/json'
+ - name: "X-Operation"
+ value: 'c'
+ - name: "X-Table"
+ value: 'public.products'
+ - name: "X-Key"
+ value: '{"id":3}'
+ - name: "X-Component-Headers"
+ value: 'op= key='
+ body:
+ data: '{"id":3,"name":"Juice","price":2.10,"added":20454}'
+ - http:
+ server: "httpServer"
+ sendResponse:
+ response:
+ status: 200
+ reasonPhrase: "OK"
+ version: "HTTP/1.1"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-route.yaml
b/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-route.yaml
new file mode 100644
index 000000000..8fd401e11
--- /dev/null
+++
b/tests/camel-kamelets-itest/src/test/resources/database/debezium/debezium-postgresql-source-route.yaml
@@ -0,0 +1,54 @@
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+- route:
+ from:
+ uri: "kamelet:debezium-postgresql-source"
+ parameters:
+ serverName: "{{db.serverName}}"
+ serverPort: "{{db.serverPort}}"
+ databaseName: "testdb"
+ username: "test"
+ password: "test"
+ topicPrefix: "shop"
+ tableIncludeList: "public.products"
+ offsetStorageFileName: "{{db.offsetFile}}"
+ steps:
+ # what the Kamelet gives: the Java type of the body, the Content-Type
and its headers
+ - setHeader:
+ name: X-Body-Type
+ simple: "${body.getClass().getName()}"
+ - setHeader:
+ name: X-Content-Type
+ simple: "ct=${header.Content-Type}"
+ - setHeader:
+ name: X-Operation
+ simple: "${header.debezium-operation}"
+ - setHeader:
+ name: X-Table
+ simple: "${header.debezium-table}"
+ - setHeader:
+ name: X-Key
+ simple: "${header.debezium-key}"
+ # the headers of the component are removed
+ - setHeader:
+ name: X-Component-Headers
+ simple: "op=${header.CamelDebeziumOperation}
key=${header.CamelDebeziumKey}"
+ - setHeader:
+ name: X-Run
+ simple: "{{run.name}}"
+ - to:
+ uri: "{{http.sink.url}}/result"
diff --git
a/tests/camel-kamelets-itest/src/test/resources/database/debezium/init.sql
b/tests/camel-kamelets-itest/src/test/resources/database/debezium/init.sql
new file mode 100644
index 000000000..2d4b1f2f8
--- /dev/null
+++ b/tests/camel-kamelets-itest/src/test/resources/database/debezium/init.sql
@@ -0,0 +1,3 @@
+CREATE TABLE products (id INTEGER PRIMARY KEY, name VARCHAR(100), price
NUMERIC(10,2), added DATE, note VARCHAR(100));
+INSERT INTO products VALUES (1, 'Tea', 4.50, '2026-01-01', 'not sent');
+CREATE TABLE notes (id INTEGER PRIMARY KEY, text VARCHAR(100));
diff --git a/tests/camel-kamelets-itest/src/test/resources/database/sql.groovy
b/tests/camel-kamelets-itest/src/test/resources/database/sql.groovy
index ce2ee4902..21ab02ffc 100644
--- a/tests/camel-kamelets-itest/src/test/resources/database/sql.groovy
+++ b/tests/camel-kamelets-itest/src/test/resources/database/sql.groovy
@@ -41,6 +41,10 @@ switch (name) {
container.copyFileToContainer(Transferable.of(sql), file)
command = ['bash', '-c', name + ' -utest -ptest testdb --batch
--skip-column-names < ' + file]
break
+ case 'postgres':
+ container.copyFileToContainer(Transferable.of(sql), file)
+ command = ['psql', '-U', 'test', '-d', 'testdb', '-v',
'ON_ERROR_STOP=1', '-q', '-At', '-F', '|', '-f', file]
+ break
case 'sqlserver':
container.copyFileToContainer(Transferable.of('SET NOCOUNT ON\n' +
sql), file)
command = ['/opt/mssql-tools18/bin/sqlcmd', '-S', 'localhost', '-U',
'sa', '-P', 'Citrus-Test-1', '-C', '-b',