This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch auto-pick-59461-branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 708affa17ca branch-4.0: [memorygc](lrucache) do not prune memory if
prune is disabled #59478 (#59501)
add 3f63d1ceea0 [Feature](Streaming Job) Extend streaming job to support
Postgres synchronization (#59461)
No new revisions were added by this update.
Summary of changes:
.github/workflows/build-extension.yml | 40 +-
be/src/runtime/cdc_client_mgr.cpp | 20 +-
build.sh | 1 +
.../postgresql/init/01-create-schema.sql | 1 +
.../postgresql/postgresql-14.yaml.tpl | 8 +
.../apache/doris/job/cdc/DataSourceConfigKeys.java | 1 +
.../apache/doris/job/common/DataSourceType.java | 3 +-
.../streaming/DataSourceConfigValidator.java | 4 +-
.../insert/streaming/StreamingInsertJob.java | 54 +-
.../streaming/StreamingJobSchedulerTask.java | 2 +-
.../insert/streaming/StreamingMultiTblTask.java | 12 +-
.../apache/doris/job/offset/jdbc/JdbcOffset.java | 8 +
.../job/offset/jdbc/JdbcSourceOffsetProvider.java | 53 +-
.../apache/doris/job/util/StreamingJobUtils.java | 66 +-
.../parser/LogicalPlanBuilderForEncryption.java | 19 +-
.../doris/nereids/parser/EncryptSQLTest.java | 66 +-
fs_brokers/cdc_client/build.sh | 2 +-
fs_brokers/cdc_client/pom.xml | 5 +
.../apache/doris/cdcclient/common/Constants.java | 2 +
.../org/apache/doris/cdcclient/common/Env.java | 81 ++-
.../cdcclient/controller/ClientController.java | 16 +-
.../doris/cdcclient/model/rest/RestResponse.java | 2 +
.../cdcclient/service/PipelineCoordinator.java | 115 +++-
.../doris/cdcclient/sink/DorisBatchStreamLoad.java | 3 +
.../deserialize/DebeziumJsonDeserializer.java | 43 +-
.../doris/cdcclient/source/factory/DataSource.java | 3 +-
.../source/factory/SourceReaderFactory.java | 4 +-
.../source/reader/JdbcIncrementalSourceReader.java | 730 +++++++++++++++++++++
.../cdcclient/source/reader/SourceReader.java | 31 +-
.../cdcclient/source/reader/SplitRecords.java | 21 +-
.../source/reader/mysql/MySqlSourceReader.java | 278 +++++---
.../reader/postgres/PostgresSourceReader.java | 388 +++++++++++
.../apache/doris/cdcclient/utils/ConfigUtil.java | 164 ++---
.../source/PostgresConnectionPoolFactory.java | 58 ++
.../jdbc/test_pg_jdbc_catalog.out | 2 +
.../cdc/test_streaming_mysql_job_exclude.out | 14 +-
...sql_job.out => test_streaming_postgres_job.out} | 0
.../cdc/test_streaming_postgres_job_all_type.out | 39 ++
.../cdc/test_streaming_postgres_job_priv.out} | 4 +-
regression-test/framework/pom.xml | 5 +
.../cdc/test_streaming_mysql_job_exclude.groovy | 104 ++-
...b.groovy => test_streaming_postgres_job.groovy} | 88 +--
.../test_streaming_postgres_job_all_type.groovy | 143 ++++
...oovy => test_streaming_postgres_job_dup.groovy} | 45 +-
.../cdc/test_streaming_postgres_job_priv.groovy | 150 +++++
45 files changed, 2465 insertions(+), 433 deletions(-)
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/JdbcIncrementalSourceReader.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/postgres/PostgresSourceReader.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/flink/cdc/connectors/postgres/source/PostgresConnectionPoolFactory.java
copy
regression-test/data/job_p0/streaming_job/cdc/{test_streaming_mysql_job.out =>
test_streaming_postgres_job.out} (100%)
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_postgres_job_all_type.out
copy regression-test/data/{bitmap_functions/test_bitmap_max.out =>
job_p0/streaming_job/cdc/test_streaming_postgres_job_priv.out} (92%)
copy
regression-test/suites/job_p0/streaming_job/cdc/{test_streaming_mysql_job.groovy
=> test_streaming_postgres_job.groovy} (65%)
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_all_type.groovy
copy
regression-test/suites/job_p0/streaming_job/cdc/{test_streaming_mysql_job_dup.groovy
=> test_streaming_postgres_job_dup.groovy} (63%)
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_priv.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
