This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar-connectors.git
The following commit(s) were added to refs/heads/master by this push:
new c762d7e [improve][build] Improve the gradle build to use similar best
practices as apache/pulsar build (#10)
c762d7e is described below
commit c762d7ef118f7eb165c22bf2681ce801893558f9
Author: Lari Hotari <[email protected]>
AuthorDate: Thu Apr 2 21:05:15 2026 +0300
[improve][build] Improve the gradle build to use similar best practices as
apache/pulsar build (#10)
---
.github/actions/setup-gradle/action.yml | 71 ++++
.github/actions/tune-runner-vm/action.yml | 80 +++++
.github/workflows/ci.yaml | 41 ++-
.gitignore | 1 +
.ratignore | 48 +++
README.md | 37 +-
aerospike/build.gradle.kts | 3 +-
alluxio/build.gradle.kts | 27 +-
.../apache/pulsar/io/alluxio/sink/AlluxioSink.java | 22 +-
.../pulsar/io/alluxio/sink/AlluxioSinkTest.java | 66 ++--
aws/build.gradle.kts | 4 +
azure-data-explorer/build.gradle.kts | 3 +-
.../conventions}/build.gradle.kts | 19 +-
...-connectors.code-quality-conventions.gradle.kts | 29 +-
.../pulsar-connectors.java-conventions.gradle.kts | 227 +++++++++++++
.../pulsar-connectors.nar-conventions.gradle.kts | 143 ++++++++
...pulsar-connectors.shadow-conventions.gradle.kts | 57 ++++
.../settings.gradle.kts | 19 +-
build.gradle.kts | 301 +++--------------
canal/build.gradle.kts | 3 +-
cassandra/build.gradle.kts | 3 +-
debezium/core/build.gradle.kts | 3 +
debezium/mongodb/build.gradle.kts | 3 +-
debezium/mssql/build.gradle.kts | 3 +-
debezium/mysql/build.gradle.kts | 3 +-
debezium/oracle/build.gradle.kts | 3 +-
debezium/postgres/build.gradle.kts | 3 +-
distribution/io/build.gradle.kts | 6 +-
docker/pulsar-all/build.gradle.kts | 6 +-
docs/build.gradle.kts | 34 +-
gradle.properties => docs/pulsar-io-gen.sh | 19 +-
.../pulsar/io/docs/ConnectorDocGenerator.java | 18 +-
dynamodb/build.gradle.kts | 3 +-
elastic-search/build.gradle.kts | 3 +-
file/build.gradle.kts | 3 +-
gradle.properties | 4 +
gradle/libs.versions.toml | 375 +++++++++------------
hbase/build.gradle.kts | 3 +-
hdfs3/build.gradle.kts | 3 +-
http/build.gradle.kts | 3 +-
influxdb/build.gradle.kts | 3 +-
jdbc/clickhouse/build.gradle.kts | 3 +-
jdbc/core/build.gradle.kts | 4 +
jdbc/mariadb/build.gradle.kts | 3 +-
jdbc/openmldb/build.gradle.kts | 3 +-
jdbc/postgres/build.gradle.kts | 3 +-
jdbc/sqlite/build.gradle.kts | 3 +-
kafka-connect-adaptor-nar/build.gradle.kts | 3 +-
kafka-connect-adaptor/build.gradle.kts | 3 +
kafka/build.gradle.kts | 19 +-
kinesis-kpl-shaded/build.gradle.kts | 12 +-
kinesis/build.gradle.kts | 7 +-
mongo/build.gradle.kts | 3 +-
netty/build.gradle.kts | 3 +-
nsq/build.gradle.kts | 3 +-
.../build.gradle.kts | 24 +-
rabbitmq/build.gradle.kts | 3 +-
redis/build.gradle.kts | 3 +-
settings.gradle.kts | 11 +-
solr/build.gradle.kts | 5 +-
src/license-header.txt | 16 +
61 files changed, 1199 insertions(+), 640 deletions(-)
diff --git a/.github/actions/setup-gradle/action.yml
b/.github/actions/setup-gradle/action.yml
new file mode 100644
index 0000000..ceac384
--- /dev/null
+++ b/.github/actions/setup-gradle/action.yml
@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Setup Gradle
+description: Sets up Gradle with Develocity or public build scan publishing by
default
+inputs:
+ develocity-access-key:
+ description: 'Develocity access key for authenticated build scans'
+ required: false
+ default: ''
+ build-scan-publish:
+ description: 'Whether to publish build scans or use Develocity when the
access key is set'
+ required: false
+ default: 'true'
+ cache-read-only:
+ description: 'Whether the Gradle cache is read-only'
+ required: false
+ default: ${{ github.event.repository != null && github.ref_name !=
github.event.repository.default_branch }}
+ add-job-summary:
+ description: 'When to add a job summary'
+ required: false
+ default: 'always'
+runs:
+ using: composite
+ steps:
+ - name: Set Develocity Project ID and configure custom settings
+ if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish
== 'true' }}
+ shell: bash
+ run: |
+ mkdir -p ~/.gradle
+ touch ~/.gradle/gradle.properties
+ grep -q 'systemProp.develocity.projectId=' ~/.gradle/gradle.properties
|| echo systemProp.develocity.projectId=pulsar >> ~/.gradle/gradle.properties
+ grep -q 'systemProp.scan.uploadInBackground='
~/.gradle/gradle.properties || echo systemProp.scan.uploadInBackground=false >>
~/.gradle/gradle.properties
+
+ - name: Setup Gradle with Develocity
+ if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish
== 'true' }}
+ uses:
gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f
+ with:
+ develocity-injection-enabled: true
+ develocity-url: https://develocity.apache.org
+ # expected format is develocity.apache.org:<access-key>
+ develocity-access-key: ${{ inputs.develocity-access-key }}
+ build-scan-publish: ${{ inputs.build-scan-publish }}
+ cache-read-only: ${{ inputs.cache-read-only }}
+ add-job-summary: ${{ inputs.add-job-summary }}
+
+ - name: Setup Gradle
+ if: ${{ !(inputs.develocity-access-key != '' &&
inputs.build-scan-publish == 'true') }}
+ uses:
gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f
+ with:
+ build-scan-publish: ${{ inputs.build-scan-publish }}
+ build-scan-terms-of-use-url: 'https://gradle.com/terms-of-service'
+ build-scan-terms-of-use-agree: 'yes'
+ cache-read-only: ${{ inputs.cache-read-only }}
+ add-job-summary: ${{ inputs.add-job-summary }}
\ No newline at end of file
diff --git a/.github/actions/tune-runner-vm/action.yml
b/.github/actions/tune-runner-vm/action.yml
new file mode 100644
index 0000000..00eb084
--- /dev/null
+++ b/.github/actions/tune-runner-vm/action.yml
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Tune Runner VM performance
+description: tunes the GitHub Runner VM operation system
+runs:
+ using: composite
+ steps:
+ - shell: bash
+ run: |
+ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+ echo "::group::Configure and tune OS"
+ # Ensure that reverse lookups for current hostname are handled
properly
+ # Add the current IP address, long hostname and short hostname
record to /etc/hosts file
+ echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' |
cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+
+ # The default vm.swappiness setting is 60 which has a tendency to
start swapping when memory
+ # consumption is high.
+ # Set vm.swappiness=1 to avoid swapping and allow high RAM usage
+ echo 1 | sudo tee /proc/sys/vm/swappiness
+
+ # use "madvise" Linux Transparent HugePages (THP) setting
+ #
https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
+ # "madvise" is generally a better option than the default "always"
setting
+ # recommendation from
https://netflixtechblog.com/bending-pause-times-to-your-will-with-generational-zgc-256629c9386b
+ echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
+ echo advise | sudo tee
/sys/kernel/mm/transparent_hugepage/shmem_enabled
+ echo defer | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
+ echo 1 | sudo tee
/sys/kernel/mm/transparent_hugepage/khugepaged/defrag
+
+ # tune filesystem mount options,
https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
+ # commit=999999, effectively disables automatic syncing to disk
(default is every 5 seconds)
+ # nobarrier/barrier=0, loosen data consistency on system crash (no
negative impact to empheral CI nodes)
+ sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
+ if mountpoint -q /mnt; then
+ sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt ||
true
+ fi
+ # disable discard/trim at device level since remount with
nodiscard doesn't seem to be effective
+ # https://www.spinics.net/lists/linux-ide/msg52562.html
+ for i in /sys/block/sd*/queue/discard_max_bytes; do
+ echo 0 | sudo tee $i
+ done
+ # disable unnecessary timers
+ sudo systemctl stop fstrim.timer fstrim.service \
+ podman-auto-update.timer sysstat-collect.timer
sysstat-summary.timer \
+ phpsessionclean.timer man-db.timer motd-news.timer \
+ dpkg-db-backup.timer e2scrub_all.timer \
+ update-notifier-download.timer update-notifier-motd.timer || true
+
+ # stop unnecessary services
+ sudo systemctl stop php8.3-fpm.service ModemManager.service \
+ multipathd.socket multipathd.service udisks2.service
walinuxagent.service || true
+
+ echo '::endgroup::'
+
+ # show memory
+ echo "::group::Available Memory"
+ free -m
+ echo '::endgroup::'
+ # show disk
+ echo "::group::Available diskspace"
+ df -BM
+ echo "::endgroup::"
+ fi
\ No newline at end of file
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 838b667..424363a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -40,17 +40,26 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-java@v4
+ - uses: actions/checkout@v6
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
+
+ - uses: actions/setup-java@v5
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.JDK_VERSION }}
- - uses:
gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c
- - name: Build all modules
- run: ./gradlew build -x test
- - name: License check (RAT)
- run: ./gradlew rat
+ - name: Setup Gradle
+ uses: ./.github/actions/setup-gradle
+ with:
+ develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+ cache-read-only: false
+
+ - name: Build and check licenses
+ run: >-
+ ./gradlew -x test -x nar build rat spotlessCheck
+ --no-configuration-cache
tests:
name: Tests - ${{ matrix.name }}
@@ -64,18 +73,26 @@ jobs:
- name: Connectors
tasks: test
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-java@v4
+ - uses: actions/checkout@v6
+
+ - name: Tune Runner VM
+ uses: ./.github/actions/tune-runner-vm
+
+ - uses: actions/setup-java@v5
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.JDK_VERSION }}
- - uses:
gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c
+
+ - name: Setup Gradle
+ uses: ./.github/actions/setup-gradle
+ with:
+ develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
- name: Run tests
- run: ./gradlew ${{ matrix.tasks }}
+ run: ./gradlew ${{ matrix.tasks }} -PtestFailFast=true
- name: Upload test reports
if: failure()
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v7
with:
name: ${{ matrix.name }}-test-reports
path: '**/build/reports/tests/'
diff --git a/.gitignore b/.gitignore
index 91bd581..33093cf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
# Gradle
.gradle/
+.kotlin/
build/
**/build/
diff --git a/.ratignore b/.ratignore
new file mode 100644
index 0000000..ab06f40
--- /dev/null
+++ b/.ratignore
@@ -0,0 +1,48 @@
+# Build artifacts
+**/build/**
+**/target/**
+# Gradle files
+.gradle/**
+gradle/wrapper/**
+**/.gradle/**
+**/.kotlin/**
+**/gradle/wrapper/**
+gradlew
+gradlew.bat
+gradle/libs.versions.toml
+# Generated Flatbuffer files (Kinesis)
+**/org/apache/pulsar/io/kinesis/fbs/*.java
+# Services files
+**/META-INF/services/*
+# Certificates and keys
+**/*.crt
+**/*.key
+**/*.csr
+**/*.pem
+**/*.srl
+**/certificate-authority/serial
+**/certificate-authority/index.txt
+**/*.json
+**/*.txt
+# Project/IDE files
+**/*.md
+.github/**
+**/*.nar
+**/.gitignore
+**/.gitattributes
+**/*.iml
+**/.classpath
+**/.project
+**/.settings
+**/.idea/**
+**/.vscode/**
+# Avro schemas
+**/*.avsc
+# Patch files
+**/*.patch
+# Hidden directories
+.*/**
+# Test output
+**/test-output/**
+# Log files
+**/*.log
diff --git a/README.md b/README.md
index 6ecb271..ea0694f 100644
--- a/README.md
+++ b/README.md
@@ -66,26 +66,48 @@ mounting them into the `apachepulsar/pulsar` Docker image.
|-----------|-------------|
| Kafka Connect Adaptor | Run Kafka Connect connectors on Pulsar |
+## Prerequisites
+
+- **JDK 17** or later — e.g. [Eclipse
Temurin](https://adoptium.net/en-GB/temurin/releases?version=17&os=any&arch=any)
+ or [Amazon
Corretto](https://docs.aws.amazon.com/corretto/latest/corretto-17-ug/what-is-corretto-17.html)
+
+> **Note**: This project includes a [Gradle
Wrapper](https://docs.gradle.org/current/userguide/gradle_wrapper.html)
+> so no separate Gradle installation is needed. Use `./gradlew` on Linux/macOS
and `gradlew.bat` on Windows.
+
## Building
+Compile and assemble all modules:
+
```bash
-./gradlew build -x test
+./gradlew assemble
```
-To build all connector NARs:
+NAR files are produced under each connector's `build/libs/` directory.
+
+Build a specific connector:
```bash
-./gradlew build -x test
+./gradlew :elastic-search:assemble
```
-NAR files are produced under each connector's `build/libs/` directory.
-
-To build the distribution tarball containing all connector NARs:
+Build the distribution package containing all connector NARs:
```bash
./gradlew :distribution:pulsar-io-distribution:assemble
```
+Check source code license headers:
+
+```bash
+./gradlew rat spotlessCheck
+```
+
+Auto-fix license headers:
+
+```bash
+./gradlew spotlessApply
+```
+
## Running Tests
```bash
@@ -94,6 +116,9 @@ To build the distribution tarball containing all connector
NARs:
# Specific connector
./gradlew :elastic-search:test
+
+# Specific test class
+./gradlew :elastic-search:test --tests "ElasticSearchSinkTests"
```
## Using Connectors
diff --git a/aerospike/build.gradle.kts b/aerospike/build.gradle.kts
index eae094f..9fb452f 100644
--- a/aerospike/build.gradle.kts
+++ b/aerospike/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/alluxio/build.gradle.kts b/alluxio/build.gradle.kts
index be406b5..9d2be5b 100644
--- a/alluxio/build.gradle.kts
+++ b/alluxio/build.gradle.kts
@@ -18,14 +18,33 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
+
+val alluxioVersion = "2.9.4"
+
+// Alluxio requires older versions of netty, grpc, and jetty than the shared
platform provides.
+// Exclude these BOMs from the enforced platform so the alluxio-specific
versions below can apply.
+pulsarConnectorsDependencies {
+ exclude(libs.jetty.bom)
+ exclude(libs.netty.bom)
+ exclude(libs.grpc.bom)
+}
+
dependencies {
+ // Alluxio-compatible BOMs — these override the shared platform versions.
+ implementation(enforcedPlatform(libs.jetty9.bom))
+ implementation(enforcedPlatform("io.netty:netty-bom:4.1.100.Final"))
+ implementation(enforcedPlatform("io.grpc:grpc-bom:1.37.0"))
+
implementation(libs.pulsar.io.core)
- implementation("org.alluxio:alluxio-core-client-fs:2.9.3")
+ implementation("org.alluxio:alluxio-core-client-fs:$alluxioVersion")
implementation(libs.jackson.dataformat.yaml)
implementation(libs.guava)
testImplementation(libs.pulsar.client)
- testImplementation("org.alluxio:alluxio-minicluster:2.9.3")
-}
+ testImplementation("org.alluxio:alluxio-minicluster:$alluxioVersion") {
+ exclude(group = "org.glassfish", module = "javax.el")
+ }
+}
\ No newline at end of file
diff --git
a/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java
b/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java
index 3b72dc9..bde3cad 100644
--- a/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java
+++ b/alluxio/src/main/java/org/apache/pulsar/io/alluxio/sink/AlluxioSink.java
@@ -151,11 +151,7 @@ public class AlluxioSink implements Sink<GenericObject> {
} catch (AlluxioException | IOException e) {
log.error("Unable to flush records to alluxio.", e);
failRecords();
- try {
- deleteTmpFile();
- } catch (AlluxioException | IOException e1) {
- log.error("Failed to delete tmp cache file.", e);
- }
+ deleteTmpFile();
break;
}
case FILE_COMMITTED:
@@ -224,10 +220,11 @@ public class AlluxioSink implements Sink<GenericObject> {
}
private void closeAndCommitTmpFile() throws AlluxioException, IOException {
- // close the tmpFile
- if (fileOutStream != null) {
- fileOutStream.close();
+ if (fileOutStream == null) {
+ return;
}
+ // close the tmpFile
+ fileOutStream.close();
// commit the tmpFile
String filePrefix = alluxioSinkConfig.getFilePrefix();
String fileExtension = alluxioSinkConfig.getFileExtension();
@@ -240,9 +237,14 @@ public class AlluxioSink implements Sink<GenericObject> {
lastRotationTime = System.currentTimeMillis();
}
- private void deleteTmpFile() throws AlluxioException, IOException {
- if (!tmpFilePath.equals("")) {
+ private void deleteTmpFile() {
+ if (tmpFilePath == null || tmpFilePath.isEmpty()) {
+ return;
+ }
+ try {
fileSystem.delete(new AlluxioURI(tmpFilePath));
+ } catch (Exception e) {
+ log.warn("Failed to delete tmp file {}", tmpFilePath, e);
}
}
diff --git
a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java
b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java
index 2f777b6..b1b1d0f 100644
---
a/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java
+++
b/alluxio/src/test/java/org/apache/pulsar/io/alluxio/sink/AlluxioSinkTest.java
@@ -27,6 +27,8 @@ import alluxio.client.file.URIStatus;
import alluxio.conf.Configuration;
import alluxio.conf.PropertyKey;
import alluxio.master.LocalAlluxioCluster;
+import alluxio.security.authentication.AuthType;
+import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -47,6 +49,7 @@ import org.mockito.Mock;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.testng.Assert;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
@@ -64,6 +67,7 @@ public class AlluxioSinkTest {
protected Map<String, Object> map;
protected AlluxioSink sink;
protected LocalAlluxioCluster cluster;
+ protected String alluxioDir;
@Mock
protected Record<GenericObject> mockRecord;
@@ -74,7 +78,7 @@ public class AlluxioSinkTest {
static GenericRecord fooBar;
@BeforeClass
- public static void init() {
+ public void init() throws Exception {
valueSchema = Schema.JSON(Foobar.class);
genericSchema = Schema.generic(valueSchema.getSchemaInfo());
fooBar = genericSchema.newRecordBuilder()
@@ -83,25 +87,30 @@ public class AlluxioSinkTest {
.set("age", 20)
.build();
kvSchema = Schema.KeyValue(Schema.STRING, genericSchema,
KeyValueEncodingType.SEPARATED);
- }
- @BeforeMethod
- public final void setUp() throws Exception {
try {
cluster = setupSingleMasterCluster();
} catch (java.util.concurrent.TimeoutException e) {
throw new org.testng.SkipException(
"Skipping test: Alluxio local cluster failed to start
within timeout", e);
}
+ }
+
+ @AfterClass
+ public void destroyCluster() throws Exception {
+ if (cluster != null) {
+ cluster.stop();
+ }
+ }
+
+ @BeforeMethod
+ public final void setUp(Method method) throws Exception {
+ alluxioDir = "/" + method.getName();
map = new HashMap<>();
- // alluxioMasterHost should be set via LocalAlluxioCluster#getHostname
- // instead of using a fixed value "localhost", since it seems that
- // LocalAlluxioCluster may bind other address than localhost
- // when the node has multiple network interfaces.
map.put("alluxioMasterHost", cluster.getHostname());
map.put("alluxioMasterPort", cluster.getMasterRpcPort());
- map.put("alluxioDir", "/pulsar");
+ map.put("alluxioDir", alluxioDir);
map.put("filePrefix", "prefix");
map.put("schemaEnable", "true");
@@ -131,8 +140,9 @@ public class AlluxioSinkTest {
@AfterMethod
public void tearDown() throws Exception {
- if (cluster != null) {
- cluster.stop();
+ if (sink != null) {
+ sink.close();
+ sink = null;
}
}
@@ -143,8 +153,6 @@ public class AlluxioSinkTest {
map.put("lineSeparator", "\n");
map.put("rotationRecords", "100");
- String alluxioDir = "/pulsar";
-
sink = new AlluxioSink();
sink.open(map, mockSinkContext);
@@ -156,8 +164,6 @@ public class AlluxioSinkTest {
String alluxioTmpDir = FilenameUtils.concat(alluxioDir, "tmp");
AlluxioURI alluxioTmpURI = new AlluxioURI(alluxioTmpDir);
Assert.assertTrue(client.exists(alluxioTmpURI));
-
- sink.close();
}
@Test
@@ -167,9 +173,6 @@ public class AlluxioSinkTest {
map.put("lineSeparator", "\n");
map.put("rotationRecords", "1");
map.put("writeType", "THROUGH");
- map.put("alluxioDir", "/pulsar");
-
- String alluxioDir = "/pulsar";
sink = new AlluxioSink();
sink.open(map, mockSinkContext);
@@ -201,11 +204,9 @@ public class AlluxioSinkTest {
for (String path : pathList) {
if (path.contains("tmp")) {
- // Ensure that the temporary file is rotated and the directory
is empty
- Assert.assertEquals(path, "/pulsar/tmp");
+ Assert.assertEquals(path, alluxioDir + "/tmp");
} else {
- // Ensure that all rotated files conform the naming convention
- Assert.assertTrue(path.startsWith("/pulsar/TopicA-"));
+ Assert.assertTrue(path.startsWith(alluxioDir + "/TopicA-"));
}
}
@@ -228,21 +229,34 @@ public class AlluxioSinkTest {
for (String path : pathList) {
if (path.contains("tmp")) {
- Assert.assertEquals(path, "/pulsar/tmp");
+ Assert.assertEquals(path, alluxioDir + "/tmp");
} else {
- Assert.assertTrue(path.startsWith("/pulsar/TopicA-"));
+ Assert.assertTrue(path.startsWith(alluxioDir + "/TopicA-"));
}
}
-
- sink.close();
}
private LocalAlluxioCluster setupSingleMasterCluster() throws Exception {
// Setup and start the local alluxio cluster
+ log.info("Configuring local Alluxio cluster");
LocalAlluxioCluster cluster = new LocalAlluxioCluster();
cluster.initConfiguration(getTestName(getClass().getSimpleName(),
"test"));
Configuration.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT,
WriteType.MUST_CACHE);
+ // Disable auth handshake overhead
+ Configuration.set(PropertyKey.SECURITY_AUTHENTICATION_TYPE,
AuthType.NOSASL);
+
Configuration.set(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED, false);
+ // Disable unnecessary services
+ Configuration.set(PropertyKey.USER_METRICS_COLLECTION_ENABLED, false);
+ Configuration.set(PropertyKey.MASTER_AUDIT_LOGGING_ENABLED, false);
+ Configuration.set(PropertyKey.MASTER_DAILY_BACKUP_ENABLED, false);
+
Configuration.set(PropertyKey.MASTER_STARTUP_BLOCK_INTEGRITY_CHECK_ENABLED,
false);
+ // Minimal safe mode wait — in a local minicluster the worker
registers almost instantly.
+ // A longer wait (e.g. 5s) causes "master is in safe mode" failures
when tests write right
+ // after startup.
+ Configuration.set(PropertyKey.MASTER_WORKER_CONNECT_WAIT_TIME,
"500ms");
+ log.info("Starting local Alluxio cluster");
cluster.start();
+ log.info("Alluxio cluster started");
return cluster;
}
diff --git a/aws/build.gradle.kts b/aws/build.gradle.kts
index b845607..81fc5e0 100644
--- a/aws/build.gradle.kts
+++ b/aws/build.gradle.kts
@@ -17,6 +17,10 @@
* under the License.
*/
+plugins {
+ id("pulsar-connectors.java-conventions")
+}
+
dependencies {
implementation(libs.pulsar.io.core)
implementation(libs.gson)
diff --git a/azure-data-explorer/build.gradle.kts
b/azure-data-explorer/build.gradle.kts
index 7fcedd5..0f0662b 100644
--- a/azure-data-explorer/build.gradle.kts
+++ b/azure-data-explorer/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
nar {
narId.set("pulsar-io-azuredataexplorer")
diff --git a/alluxio/build.gradle.kts b/build-logic/conventions/build.gradle.kts
similarity index 68%
copy from alluxio/build.gradle.kts
copy to build-logic/conventions/build.gradle.kts
index be406b5..bc3d07a 100644
--- a/alluxio/build.gradle.kts
+++ b/build-logic/conventions/build.gradle.kts
@@ -18,14 +18,17 @@
*/
plugins {
- alias(libs.plugins.nar)
+ `kotlin-dsl`
}
-dependencies {
- implementation(libs.pulsar.io.core)
- implementation("org.alluxio:alluxio-core-client-fs:2.9.3")
- implementation(libs.jackson.dataformat.yaml)
- implementation(libs.guava)
- testImplementation(libs.pulsar.client)
- testImplementation("org.alluxio:alluxio-minicluster:2.9.3")
+dependencies {
+ implementation(libs.plugins.shadow.get().let {
+ "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}"
+ })
+ implementation(libs.plugins.spotless.get().let {
+ "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}"
+ })
+ implementation(libs.plugins.nar.get().let {
+ "${it.pluginId}:${it.pluginId}.gradle.plugin:${it.version}"
+ })
}
diff --git a/hdfs3/build.gradle.kts
b/build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts
similarity index 50%
copy from hdfs3/build.gradle.kts
copy to
build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts
index 1f20430..ad2ffae 100644
--- a/hdfs3/build.gradle.kts
+++
b/build-logic/conventions/src/main/kotlin/pulsar-connectors.code-quality-conventions.gradle.kts
@@ -18,17 +18,24 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("com.diffplug.spotless")
}
-dependencies {
- implementation(libs.pulsar.io.core)
- implementation(libs.jackson.databind)
- implementation(libs.jackson.dataformat.yaml)
- implementation(libs.commons.collections4)
- implementation(libs.hadoop.client) {
- exclude(group = "org.slf4j")
- exclude(group = "log4j")
+
+// ── License header check (Spotless) ────────────────────────────────────────
+val asfLicenseHeader = rootProject.file("src/license-header.txt").readText()
+val asfLicenseHeaderJava = "/*\n" + asfLicenseHeader.lines()
+ .map { " * $it".trimEnd() }
+ .joinToString("\n") + "/\n"
+
+configure<com.diffplug.gradle.spotless.SpotlessExtension> {
+ java {
+ targetExclude(
+ "**/generated/**",
+ "**/generated-sources/**",
+ // Generated FlatBuffers files (Kinesis)
+ "**/org/apache/pulsar/io/kinesis/fbs/*.java",
+ "build/**",
+ )
+ licenseHeader(asfLicenseHeaderJava,
"(\\n|package|import|public|class|module) ?")
}
- implementation(libs.bcprov.jdk18on)
- implementation(libs.jakarta.activation.api)
}
diff --git
a/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts
b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts
new file mode 100644
index 0000000..a118e33
--- /dev/null
+++
b/build-logic/conventions/src/main/kotlin/pulsar-connectors.java-conventions.gradle.kts
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+ `java-library`
+ id("pulsar-connectors.code-quality-conventions")
+}
+
+val catalog = the<VersionCatalogsExtension>().named("libs")
+
+fun lib(alias: String): Provider<MinimalExternalModuleDependency> =
+ catalog.findLibrary(alias).orElseThrow {
+ GradleException("Library alias '$alias' not found in version catalog
'libs'")
+ }
+
+// Add shared test resources (log4j2-test.xml) to the test classpath for all
modules.
+the<SourceSetContainer>()["test"].resources.srcDir(rootProject.file("gradle/test-resources"))
+
+tasks.withType<JavaCompile>().configureEach {
+ options.encoding = "UTF-8"
+ options.release.set(17)
+ options.compilerArgs.addAll(listOf("-parameters", "-Xlint:deprecation",
"-Xlint:unchecked"))
+}
+
+configurations.all {
+ // Exclude the old SLF4J 1.x bridge pulled in by transitive dependencies.
+ // Pulsar uses SLF4J 2.x with log4j-slf4j2-impl; having both causes
+ // NoSuchMethodError in Log4jLoggerFactory at test startup.
+ exclude(group = "org.apache.logging.log4j", module = "log4j-slf4j-impl")
+}
+
+// Exclude bc-fips from modules that don't need it. bc-fips's
CryptoServicesRegistrar
+// conflicts with bcprov-jdk18on's version — having both causes
NoSuchMethodError.
+val modulesUsingBcFips = setOf("kafka-connect-adaptor")
+if (project.name !in modulesUsingBcFips) {
+ configurations.all {
+ exclude(group = "org.bouncycastle", module = "bc-fips")
+ }
+}
+
+/**
+ * Configures how the shared `pulsar-connectors-dependencies` platform is
applied.
+ *
+ * By default, the platform is applied as `enforcedPlatform`, which pins
(strictly enforces) all
+ * dependency versions from the version catalog. Subprojects can customize
this behavior:
+ *
+ * ```kotlin
+ * pulsarConnectorsDependencies {
+ * // Exclude using a version catalog reference:
+ * exclude(libs.netty.bom)
+ *
+ * // Exclude using "group:module" notation:
+ * exclude("io.netty:netty-bom")
+ *
+ * // Exclude using named parameters:
+ * exclude(group = "io.netty", module = "netty-bom")
+ *
+ * // Set enforced = false to use platform() instead of
enforcedPlatform(). This makes all
+ * // version constraints non-strict: the platform versions are used only
as defaults (allowing
+ * // version omission when declaring dependencies), but can be overridden
by the module's own
+ * // enforcedPlatform or strictly-versioned dependencies.
+ * enforced = false
+ * }
+ * ```
+ */
+open class PulsarConnectorsDependenciesExtension {
+ var enforced: Boolean = true
+
+ internal val excludes: MutableList<DependencyExclusion> = mutableListOf()
+
+ fun exclude(group: String, module: String) {
+ excludes.add(DependencyExclusion(group, module))
+ }
+
+ fun exclude(dependency: Provider<MinimalExternalModuleDependency>) {
+ val dep = dependency.get()
+ excludes.add(DependencyExclusion(dep.module.group, dep.module.name))
+ }
+
+ fun exclude(notation: String) {
+ val parts = notation.split(":")
+ require(parts.size == 2) { "Expected 'group:module' format, got:
$notation" }
+ excludes.add(DependencyExclusion(parts[0], parts[1]))
+ }
+}
+
+data class DependencyExclusion(val group: String, val module: String)
+
+val pulsarConnectorsDependencies =
extensions.create<PulsarConnectorsDependenciesExtension>("pulsarConnectorsDependencies")
+
+// withDependencies runs lazily after subproject build scripts have configured
the extension.
+// This is configuration-cache compatible (unlike afterEvaluate).
+configurations["implementation"].withDependencies {
+ val platformProject = project(":pulsar-connectors-dependencies")
+ val configureAction = Action<Dependency> {
+ (this as ModuleDependency).apply {
+ pulsarConnectorsDependencies.excludes.forEach { exc ->
+ exclude(group = exc.group, module = exc.module)
+ }
+ }
+ }
+ val dep = if (pulsarConnectorsDependencies.enforced) {
+ dependencies.enforcedPlatform(platformProject, configureAction)
+ } else {
+ dependencies.platform(platformProject, configureAction)
+ }
+ add(dep)
+}
+
+dependencies {
+
+ // Resolve lz4-java capability conflict: at.yawk.lz4:lz4-java (used by
Pulsar) and
+ // org.lz4:lz4-java (used by kafka-clients) both provide the
org.lz4:lz4-java capability.
+ // Prefer at.yawk.lz4 which is the version Pulsar standardizes on.
+ configurations.all {
+
resolutionStrategy.capabilitiesResolution.withCapability("org.lz4:lz4-java") {
+ select("at.yawk.lz4:lz4-java:0")
+ }
+ }
+
+ // Annotation processing for Lombok
+ "compileOnly"(lib("lombok"))
+ "annotationProcessor"(lib("lombok"))
+ "testCompileOnly"(lib("lombok"))
+ "testAnnotationProcessor"(lib("lombok"))
+
+ // Common test dependencies
+ "testImplementation"(lib("testng"))
+ "testImplementation"(lib("mockito-core"))
+ "testImplementation"(lib("assertj-core"))
+ "testImplementation"(lib("awaitility"))
+ "testImplementation"(lib("system-lambda"))
+ "testImplementation"(lib("slf4j-api"))
+
+ // Logging runtime for tests — provides Log4j2 as the SLF4J backend.
+ // Some connectors (Alluxio minicluster, Solr embedded) require a logging
+ // implementation to be present at test runtime.
+ "testRuntimeOnly"(lib("log4j-api"))
+ "testRuntimeOnly"(lib("log4j-core"))
+ "testRuntimeOnly"(lib("log4j-slf4j2-impl"))
+ "testRuntimeOnly"(lib("jcl-over-slf4j"))
+}
+
+tasks.withType<Test>().configureEach {
+ useTestNG {
+ // TestNG group filtering: -PtestGroups=group1,group2
-PexcludedTestGroups=flaky
+ providers.gradleProperty("testGroups").orNull?.let { groups ->
+ includeGroups(*groups.split(",").map { it.trim() }.toTypedArray())
+ }
+ val excludedTestGroups =
providers.gradleProperty("excludedTestGroups").getOrElse("quarantine,flaky")
+ excludeGroups(*(excludedTestGroups.split(",").map { it.trim()
}.toTypedArray()))
+ }
+ testLogging {
+ events("passed", "skipped", "failed")
+ exceptionFormat =
org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL
+ showStackTraces = true
+ showExceptions = true
+ showCauses = true
+ showStandardStreams = providers.gradleProperty("testShowOutput")
+ .map { it.isBlank() || it.toBoolean() }.getOrElse(false)
+ }
+ maxHeapSize = "1300m"
+ maxParallelForks = 4
+ val failFastValue =
providers.gradleProperty("testFailFast").getOrElse("true").toBoolean()
+ failFast = failFastValue
+ systemProperty("testRetryCount",
providers.gradleProperty("testRetryCount").getOrElse("1"))
+ systemProperty("testFailFast", failFastValue.toString())
+ systemProperty("java.net.preferIPv4Stack", "true")
+ jvmArgs(
+ "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED",
+ "--add-opens", "java.base/java.lang=ALL-UNNAMED",
+ "--add-opens", "java.base/java.io=ALL-UNNAMED",
+ "--add-opens", "java.base/java.util=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.net=ALL-UNNAMED",
+ "--add-opens", "java.management/sun.management=ALL-UNNAMED",
+ "--add-opens",
"jdk.management/com.sun.management.internal=ALL-UNNAMED",
+ "--add-opens", "java.base/jdk.internal.platform=ALL-UNNAMED",
+ "--add-opens", "java.base/java.nio=ALL-UNNAMED",
+ "--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED",
+ "-XX:+EnableDynamicAgentLoading",
+ "-Xshare:off",
+ "-Dio.netty.tryReflectionSetAccessible=true",
+ "-Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true",
+ "-Dpulsar.allocator.pooled=true",
+ "-Dpulsar.allocator.exit_on_oom=false",
+ "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap",
+ "-Dpulsar.test.preventExit=true",
+ )
+}
+
+// Set archive names to match Maven artifactId for nested modules.
+// Skip if the project name is already qualified (starts with parent name),
+// which happens for sub-modules that use qualified names in
settings.gradle.kts
+// to avoid Gradle name clashes.
+val parentProject = project.parent
+if (parentProject != null && parentProject != rootProject &&
parentProject.parent != rootProject
+ && !project.name.startsWith(parentProject.name)) {
+
the<BasePluginExtension>().archivesName.set("${parentProject.name}-${project.name}")
+}
+
+tasks.withType<Jar>().configureEach {
+ manifest {
+ attributes(
+ "Implementation-Title" to project.name,
+ "Implementation-Version" to project.version,
+ )
+ }
+}
+
+// Add a task for viewing all configurations for all projects in a simple way
+tasks.register<DependencyReportTask>("allDependencies"){}
diff --git
a/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts
b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts
new file mode 100644
index 0000000..29d79e3
--- /dev/null
+++
b/build-logic/conventions/src/main/kotlin/pulsar-connectors.nar-conventions.gradle.kts
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Convention plugin for NAR (Nifi Archive) modules.
+// Configures platform module exclusions from runtimeClasspath, forces JAR
artifacts
+// for bundled-dependencies, and handles archive name qualification.
+
+plugins {
+ id("io.github.merlimat.nar")
+}
+
+/**
+ * Extension for customizing NAR packaging behavior.
+ *
+ * By default, Pulsar platform modules (pulsar-common, pulsar-client, etc.)
are excluded
+ * from the NAR's bundled dependencies since they are provided at runtime by
Pulsar's
+ * classloader hierarchy. Subprojects can include specific modules when needed:
+ *
+ * ```kotlin
+ * pulsarConnectorsNar {
+ * // Include pulsar-common in the NAR bundle (e.g., for SchemaInfoImpl)
+ * includePulsarModule("pulsar-common")
+ * }
+ * ```
+ */
+open class PulsarConnectorsNarExtension {
+ internal val includedPulsarModules: MutableSet<String> = mutableSetOf()
+
+ fun includePulsarModule(module: String) {
+ includedPulsarModules.add(module)
+ }
+}
+
+val pulsarConnectorsNar =
extensions.create<PulsarConnectorsNarExtension>("pulsarConnectorsNar")
+
+// NAR modules should not bundle Pulsar platform dependencies — they are
provided
+// at runtime by Pulsar's classloader hierarchy.
+// Note: pulsar-io-common is NOT in java-instance.jar (runtime-all), so it
must be
+// bundled in each NAR that uses it (e.g., IOConfigUtils).
+val defaultExcludedPulsarModules = setOf(
+ "pulsar-client-api",
+ "pulsar-client-admin-api",
+ "pulsar-client-original",
+ "pulsar-client",
+ "pulsar-common",
+ "pulsar-config-validation",
+ "bouncy-castle-bc",
+ "pulsar-functions-api",
+ "pulsar-functions-instance",
+ "pulsar-functions-proto",
+ "pulsar-functions-secrets",
+ "pulsar-functions-utils",
+ "pulsar-io-core",
+ "pulsar-metadata",
+ "pulsar-opentelemetry",
+ "managed-ledger",
+ "pulsar-package-core",
+)
+
+// Use withDependencies to defer exclusion logic until after subproject build
scripts
+// have configured the extension.
+configurations.named("runtimeClasspath") {
+ exclude(group = "org.apache.bookkeeper")
+ // Protobuf is in java-instance.jar (runtime-all), so NARs must not bundle
it.
+ // Bundling a different version causes GeneratedMessage.getUnknownFields()
conflicts.
+ exclude(group = "com.google.protobuf")
+ withDependencies {
+ val excludedModules = defaultExcludedPulsarModules -
pulsarConnectorsNar.includedPulsarModules
+ excludedModules.forEach { module ->
+ exclude(group = "org.apache.pulsar", module = module)
+ }
+ }
+}
+
+// The NAR plugin copies from runtimeClasspath which resolves project
dependencies
+// as class directories, not JARs. The NarClassLoader expects JARs in
+// META-INF/bundled-dependencies/. Force the NAR task to use JAR artifacts.
+// Use lazy resolution to avoid eagerly resolving the configuration at
configuration
+// time, which would cause configuration cache invalidation when JARs are
created.
+tasks.named("nar", Jar::class.java) {
+ val runtimeClasspath = configurations.named("runtimeClasspath")
+ into("META-INF/bundled-dependencies") {
+ from(runtimeClasspath.map { conf ->
+ conf.incoming.artifactView {
+ attributes {
+ attribute(
+ LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE,
+ objects.named(LibraryElements::class.java,
LibraryElements.JAR)
+ )
+ }
+ }.files
+ })
+ duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+ }
+}
+
+// Set NAR-specific archive name qualification for nested modules.
+val parentProject = project.parent
+if (parentProject != null && parentProject != rootProject &&
parentProject.parent != rootProject
+ && !project.name.startsWith(parentProject.name)) {
+ val qualifiedName = "${parentProject.name}-${project.name}"
+ val narExt = extensions.getByName("nar")
+ @Suppress("UNCHECKED_CAST")
+ val narIdProp = narExt.javaClass.getMethod("getNarId").invoke(narExt) as
org.gradle.api.provider.Property<String>
+ narIdProp.set(qualifiedName)
+}
+
+tasks.named<Jar>("jar") {
+ enabled = false
+}
+
+configurations {
+ named("runtimeElements") {
+ outgoing {
+ artifacts.clear()
+ artifact(tasks.named("nar"))
+ variants.clear()
+ }
+ }
+ named("apiElements") {
+ outgoing {
+ artifacts.clear()
+ artifact(tasks.named("nar"))
+ variants.clear()
+ }
+ }
+}
\ No newline at end of file
diff --git
a/build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts
b/build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts
new file mode 100644
index 0000000..7a760e5
--- /dev/null
+++
b/build-logic/conventions/src/main/kotlin/pulsar-connectors.shadow-conventions.gradle.kts
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Convention plugin for modules using the Shadow plugin.
+// Applies the shadow plugin, disables the default jar task, and makes the
+// shadow jar the primary artifact for both runtimeElements and apiElements,
+// so plain project() dependencies resolve to the shadow jar.
+
+plugins {
+ id("com.gradleup.shadow")
+}
+
+shadow {
+ addShadowVariantIntoJavaComponent.set(false)
+}
+
+tasks.named<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar>("shadowJar")
{
+ archiveClassifier.set("")
+ mergeServiceFiles()
+}
+
+tasks.named<Jar>("jar") {
+ enabled = false
+}
+
+configurations {
+ named("runtimeElements") {
+ outgoing {
+ artifacts.clear()
+ artifact(tasks.named("shadowJar"))
+ variants.clear()
+ }
+ }
+ named("apiElements") {
+ outgoing {
+ artifacts.clear()
+ artifact(tasks.named("shadowJar"))
+ variants.clear()
+ }
+ }
+}
diff --git a/debezium/mongodb/build.gradle.kts b/build-logic/settings.gradle.kts
similarity index 73%
copy from debezium/mongodb/build.gradle.kts
copy to build-logic/settings.gradle.kts
index 53c2670..ec81a01 100644
--- a/debezium/mongodb/build.gradle.kts
+++ b/build-logic/settings.gradle.kts
@@ -17,11 +17,16 @@
* under the License.
*/
-plugins {
- alias(libs.plugins.nar)
-}
-dependencies {
- implementation(libs.pulsar.io.core)
- implementation(project(":debezium:pulsar-io-debezium-core"))
- implementation(libs.debezium.connector.mongodb)
+dependencyResolutionManagement {
+ repositories {
+ gradlePluginPortal()
+ mavenCentral()
+ }
+ versionCatalogs {
+ create("libs") {
+ from(files("../gradle/libs.versions.toml"))
+ }
+ }
}
+rootProject.name = "build-logic"
+include("conventions")
diff --git a/build.gradle.kts b/build.gradle.kts
index 9aefca6..9f08278 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -17,269 +17,66 @@
* under the License.
*/
+import com.github.vlsi.gradle.git.dsl.gitignore
+import org.jetbrains.gradle.ext.copyright
+import org.jetbrains.gradle.ext.settings
+
plugins {
alias(libs.plugins.rat)
+ alias(libs.plugins.version.catalog.update)
+ alias(libs.plugins.versions)
+ alias(libs.plugins.crlf) apply false
+ alias(libs.plugins.idea.ext)
+ alias(libs.plugins.spotless) apply false // workaround for
https://github.com/diffplug/spotless/issues/2877
}
-tasks.named("rat").configure {
- val excludesProp = this.javaClass.getMethod("getExcludes").invoke(this)
- @Suppress("UNCHECKED_CAST")
- val excludes = excludesProp as MutableCollection<String>
- excludes.addAll(listOf(
- // Build artifacts
- "**/build/**",
- "**/target/**",
- // Gradle files
- ".gradle/**",
- "gradle/wrapper/**",
- "**/.gradle/**",
- "**/gradle/wrapper/**",
- // Generated Flatbuffer files (Kinesis)
- "**/org/apache/pulsar/io/kinesis/fbs/*.java",
- // Services files
- "**/META-INF/services/*",
- // Certificates and keys
- "**/*.crt",
- "**/*.key",
- "**/*.csr",
- "**/*.pem",
- "**/*.srl",
- "**/certificate-authority/serial",
- "**/certificate-authority/index.txt",
- "**/*.json",
- "**/*.txt",
- // Project/IDE files
- "**/*.md",
- ".github/**",
- "**/*.nar",
- "**/.gitignore",
- "**/.gitattributes",
- "**/*.iml",
- "**/.classpath",
- "**/.project",
- "**/.settings",
- "**/.idea/**",
- "**/.vscode/**",
- // Avro schemas
- "**/*.avsc",
- // Patch files
- "**/*.patch",
- // Hidden directories
- ".*/**",
- // Test output
- "**/test-output/**",
- // Log files
- "**/*.log",
- ))
-}
-
-val catalog = the<VersionCatalogsExtension>().named("libs")
-val pulsarConnectorsVersion =
catalog.findVersion("pulsar-connectors").get().requiredVersion
-
-allprojects {
- group = "org.apache.pulsar"
- version = pulsarConnectorsVersion
-}
-
-subprojects {
- // Platform modules use java-platform which is mutually exclusive with
java-library.
- if (project.name == "pulsar-dependencies") {
- return@subprojects
- }
-
- // Parent modules and non-Java modules that have no source code of their
own
- val skipModules = setOf("jdbc", "debezium", "distribution", "docker")
- if (project.name in skipModules && project.childProjects.isNotEmpty()) {
- return@subprojects
- }
-
- apply(plugin = "java-library")
-
- // Add shared test resources (log4j2-test.xml) to the test classpath for
all modules.
-
the<SourceSetContainer>()["test"].resources.srcDir(rootProject.file("gradle/test-resources"))
-
- tasks.withType<JavaCompile> {
- options.encoding = "UTF-8"
- options.release.set(17)
- options.compilerArgs.addAll(listOf("-parameters"))
- }
-
- configurations.all {
- // Exclude the old SLF4J 1.x bridge
- exclude(group = "org.apache.logging.log4j", module =
"log4j-slf4j-impl")
-
- // Force Jackson version to match the version catalog
- resolutionStrategy.eachDependency {
- if (requested.group.startsWith("com.fasterxml.jackson")) {
- useVersion(rootProject.libs.versions.jackson.get())
- }
- }
- }
-
- // Exclude bc-fips from modules that don't need it.
- val modulesUsingBcFips = setOf("kafka-connect-adaptor")
- if (project.name !in modulesUsingBcFips) {
- configurations.all {
- exclude(group = "org.bouncycastle", module = "bc-fips")
- }
- }
-
- dependencies {
- // Enforced platform pins all dependency versions from the version
catalog.
- "implementation"(enforcedPlatform(project(":pulsar-dependencies")))
-
- // Resolve lz4-java capability conflict
- configurations.all {
-
resolutionStrategy.capabilitiesResolution.withCapability("org.lz4:lz4-java") {
- select("at.yawk.lz4:lz4-java:0")
- }
- }
-
- // Annotation processing for Lombok
- "compileOnly"(rootProject.libs.lombok)
- "annotationProcessor"(rootProject.libs.lombok)
- "testCompileOnly"(rootProject.libs.lombok)
- "testAnnotationProcessor"(rootProject.libs.lombok)
-
- // Common test dependencies
- "testImplementation"(rootProject.libs.testng)
- "testImplementation"(rootProject.libs.mockito.core)
- "testImplementation"(rootProject.libs.assertj.core)
- "testImplementation"(rootProject.libs.awaitility)
- "testImplementation"(rootProject.libs.system.lambda)
- "testImplementation"(rootProject.libs.slf4j.api)
-
- // Logging runtime for tests — provides Log4j2 as the SLF4J backend.
- // Some connectors (Alluxio minicluster, Solr embedded) require a
logging
- // implementation to be present at test runtime.
- "testRuntimeOnly"(rootProject.libs.log4j.api)
- "testRuntimeOnly"(rootProject.libs.log4j.core)
- "testRuntimeOnly"(rootProject.libs.log4j.slf4j2.impl)
- "testRuntimeOnly"(rootProject.libs.jcl.over.slf4j)
- }
-
- tasks.withType<Test> {
- useTestNG {
- // TestNG group filtering
- providers.gradleProperty("testGroups").orNull?.let { groups ->
- includeGroups(*groups.split(",").map { it.trim()
}.toTypedArray())
- }
- val excludedTestGroups =
providers.gradleProperty("excludedTestGroups").getOrElse("quarantine,flaky")
- excludeGroups(*(excludedTestGroups.split(",").map { it.trim()
}.toTypedArray()))
- }
- maxHeapSize = "1300m"
- maxParallelForks = 4
- systemProperty("testRetryCount", System.getProperty("testRetryCount",
"1"))
- systemProperty("testFailFast", System.getProperty("testFailFast",
"true"))
- jvmArgs(
- "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED",
- "--add-opens", "java.base/java.lang=ALL-UNNAMED",
- "--add-opens", "java.base/java.io=ALL-UNNAMED",
- "--add-opens", "java.base/java.util=ALL-UNNAMED",
- "--add-opens", "java.base/sun.net=ALL-UNNAMED",
- "--add-opens", "java.management/sun.management=ALL-UNNAMED",
- "--add-opens",
"jdk.management/com.sun.management.internal=ALL-UNNAMED",
- "--add-opens", "java.base/jdk.internal.platform=ALL-UNNAMED",
- "--add-opens", "java.base/java.nio=ALL-UNNAMED",
- "--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED",
- "-XX:+EnableDynamicAgentLoading",
- "-Xshare:off",
- "-Dio.netty.tryReflectionSetAccessible=true",
- "-Dpulsar.allocator.pooled=true",
- "-Dpulsar.allocator.exit_on_oom=false",
- "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap",
- "-Dpulsar.test.preventExit=true",
- )
+versionCatalogUpdate {
+ sortByKey = false
+ keep {
+ keepUnusedVersions.set(true)
}
+}
- // Shadow JAR modules: expose the shadow JAR as a consumable configuration
so other
- // projects can depend on it via project(path = "...", configuration =
"shadowElements")
- pluginManager.withPlugin("com.gradleup.shadow") {
- val shadowElements by configurations.creating {
- isCanBeConsumed = true
- isCanBeResolved = false
- attributes {
- attribute(Usage.USAGE_ATTRIBUTE,
objects.named(Usage.JAVA_RUNTIME))
- attribute(Bundling.BUNDLING_ATTRIBUTE,
objects.named(Bundling.SHADOWED))
- }
- }
- artifacts.add("shadowElements", tasks.named("shadowJar"))
+tasks.named<com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask>("dependencyUpdates")
{
+ outputFormatter = "html"
+ rejectVersionIf {
+ val nonStable = candidate.version.contains("alpha") ||
candidate.version.contains("beta") || candidate.version.contains("rc")
+ // OpenTelemetry publishes stable releases with -alpha suffix for some
modules
+ val isOpenTelemetry = candidate.group.startsWith("io.opentelemetry")
+ nonStable && !(isOpenTelemetry && candidate.version.contains("alpha"))
}
+}
- // NAR modules should not bundle Pulsar platform dependencies — they are
provided
- // at runtime by Pulsar's classloader hierarchy.
- pluginManager.withPlugin("io.github.merlimat.nar") {
- val pulsarPlatformModules = setOf(
- "pulsar-client-api",
- "pulsar-client-admin-api",
- "pulsar-client-original",
- "pulsar-client",
- "pulsar-common",
- "pulsar-config-validation",
- "bouncy-castle-bc",
- "pulsar-functions-api",
- "pulsar-functions-instance",
- "pulsar-functions-proto",
- "pulsar-functions-secrets",
- "pulsar-functions-utils",
- "pulsar-io-core",
- "pulsar-metadata",
- "pulsar-opentelemetry",
- "managed-ledger",
- "pulsar-package-core",
- )
- configurations.named("runtimeClasspath") {
- exclude(group = "org.apache.bookkeeper")
- exclude(group = "com.google.protobuf")
- pulsarPlatformModules.forEach { module ->
- exclude(group = "org.apache.pulsar", module = module)
- }
- }
+// ── Apache RAT (Release Audit Tool) ─────────────────────────────────────────
+tasks.named<org.nosphere.apache.rat.RatTask>("rat").configure {
+ // Honour .gitignore exclusions so RAT skips untracked/generated files.
+ // Register .gitignore files as inputs so the task re-runs when they
change.
+ inputs.files(fileTree(rootDir) {
+ include("**/.gitignore")
+ exclude("**/build/**")
+ exclude("**/.gradle/**")
+ })
+ // use crlf plugin's gitignore dsl
+ gitignore(rootDir)
+ // Apply additional RAT-specific exclusions from .ratignore.
+ val ratignoreFile = rootDir.resolve(".ratignore")
+ inputs.file(ratignoreFile)
+ exclude(ratignoreFile.readLines().map { it.trim() }.filter {
it.isNotBlank() && !it.startsWith("#") })
+}
- // The NAR plugin copies from runtimeClasspath which resolves project
dependencies
- // as class directories, not JARs. The NarClassLoader expects JARs in
- // META-INF/bundled-dependencies/. Force the NAR task to use JAR
artifacts.
- val jarView = configurations.named("runtimeClasspath").get()
- .incoming.artifactView {
- attributes {
- attribute(
- LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE,
- objects.named(LibraryElements::class.java,
LibraryElements.JAR)
- )
+idea {
+ project {
+ settings {
+ // add ASL2 copyright profile to IntelliJ
+ copyright {
+ useDefault = "ASL2"
+ profiles {
+ create("ASL2") {
+ notice =
rootProject.file("src/license-header.txt").readText().trimEnd()
+ keyword = "Copyright"
+ }
}
- }.files
- tasks.named("nar", Jar::class.java) {
- into("META-INF/bundled-dependencies") {
- from(jarView)
- duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
}
}
-
- // Set archive names to match Maven artifactId for nested modules.
- val parentProject = project.parent
- if (parentProject != null && parentProject != rootProject &&
parentProject.parent != rootProject
- && !project.name.startsWith(parentProject.name)) {
- val qualifiedName = "${parentProject.name}-${project.name}"
- the<BasePluginExtension>().archivesName.set(qualifiedName)
- pluginManager.withPlugin("io.github.merlimat.nar") {
- @Suppress("UNCHECKED_CAST")
- val narExt = extensions.getByName("nar")
- val narIdProp =
narExt.javaClass.getMethod("getNarId").invoke(narExt) as Property<String>
- narIdProp.set(qualifiedName)
- }
- }
-
- tasks.withType<Jar> {
- manifest {
- attributes(
- "Implementation-Title" to project.name,
- "Implementation-Version" to project.version,
- )
- }
- }
}
-
-// Access version catalog from subprojects
-val Project.libs: org.gradle.accessors.dm.LibrariesForLibs
- get() = rootProject.extensions.getByType()
diff --git a/canal/build.gradle.kts b/canal/build.gradle.kts
index 2a20556..29397be 100644
--- a/canal/build.gradle.kts
+++ b/canal/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.common)
diff --git a/cassandra/build.gradle.kts b/cassandra/build.gradle.kts
index 8c2fab2..da76cc0 100644
--- a/cassandra/build.gradle.kts
+++ b/cassandra/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/debezium/core/build.gradle.kts b/debezium/core/build.gradle.kts
index df1cc24..746dc05 100644
--- a/debezium/core/build.gradle.kts
+++ b/debezium/core/build.gradle.kts
@@ -17,6 +17,9 @@
* under the License.
*/
+plugins {
+ id("pulsar-connectors.java-conventions")
+}
dependencies {
compileOnly(libs.pulsar.io.core)
diff --git a/debezium/mongodb/build.gradle.kts
b/debezium/mongodb/build.gradle.kts
index 53c2670..d4292a9 100644
--- a/debezium/mongodb/build.gradle.kts
+++ b/debezium/mongodb/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/debezium/mssql/build.gradle.kts b/debezium/mssql/build.gradle.kts
index 1cfcc2c..07aac7f 100644
--- a/debezium/mssql/build.gradle.kts
+++ b/debezium/mssql/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/debezium/mysql/build.gradle.kts b/debezium/mysql/build.gradle.kts
index bc79ff6..33e3048 100644
--- a/debezium/mysql/build.gradle.kts
+++ b/debezium/mysql/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/debezium/oracle/build.gradle.kts b/debezium/oracle/build.gradle.kts
index 35bf12b..dd1ef02 100644
--- a/debezium/oracle/build.gradle.kts
+++ b/debezium/oracle/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/debezium/postgres/build.gradle.kts
b/debezium/postgres/build.gradle.kts
index f49f8a8..8dc37a5 100644
--- a/debezium/postgres/build.gradle.kts
+++ b/debezium/postgres/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/distribution/io/build.gradle.kts b/distribution/io/build.gradle.kts
index 91c7721..9aa7b81 100644
--- a/distribution/io/build.gradle.kts
+++ b/distribution/io/build.gradle.kts
@@ -18,9 +18,9 @@
*/
// Distribution module — no Java compilation needed
-tasks.named("compileJava") { enabled = false }
-tasks.named("compileTestJava") { enabled = false }
-tasks.named("jar") { enabled = false }
+plugins {
+ base
+}
val pulsarVersion = project.version.toString()
diff --git a/docker/pulsar-all/build.gradle.kts
b/docker/pulsar-all/build.gradle.kts
index 5ea1aa5..448d3d1 100644
--- a/docker/pulsar-all/build.gradle.kts
+++ b/docker/pulsar-all/build.gradle.kts
@@ -18,9 +18,9 @@
*/
// Docker image module — no Java compilation needed
-tasks.named("compileJava") { enabled = false }
-tasks.named("compileTestJava") { enabled = false }
-tasks.named("jar") { enabled = false }
+plugins {
+ base
+}
val catalog = extensions.getByType<VersionCatalogsExtension>().named("libs")
val pulsarConnectorsVersion = project.version.toString()
diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts
index 62cabf8..0f89397 100644
--- a/docs/build.gradle.kts
+++ b/docs/build.gradle.kts
@@ -17,6 +17,10 @@
* under the License.
*/
+plugins {
+ id("pulsar-connectors.java-conventions")
+}
+
dependencies {
implementation(libs.pulsar.io.core)
implementation(libs.guava)
@@ -51,6 +55,34 @@ dependencies {
implementation(project(":rabbitmq"))
implementation(project(":redis"))
implementation(project(":solr"))
- implementation(project(":alluxio"))
+ implementation(project(":alluxio")) {
+ exclude("org.eclipse.jetty", "jetty-bom")
+ exclude("io.netty", "netty-bom")
+ exclude("io.grpc", "grpc-bom")
+ }
implementation(project(":azure-data-explorer"))
}
+
+val exportClasspath by tasks.registering {
+ dependsOn(tasks.classes)
+ val classpath = sourceSets.main.get().output +
configurations.runtimeClasspath.get()
+ inputs.files(classpath)
+ val outputFile = layout.buildDirectory.file("classpath.txt")
+ outputs.file(outputFile)
+ doLast {
+ outputFile.get().asFile.apply {
+ parentFile.mkdirs()
+ writeText(classpath.asPath)
+ }
+ }
+}
+
+tasks.register<JavaExec>("generateConnectorDocs") {
+ dependsOn(tasks.classes)
+ mainClass.set("org.apache.pulsar.io.docs.ConnectorDocGenerator")
+ classpath = sourceSets.main.get().output +
configurations.runtimeClasspath.get()
+ inputs.files(classpath)
+ val outputDir = layout.buildDirectory.dir("connector-docs")
+ outputs.dir(outputDir)
+ args("-o", outputDir.get().asFile.absolutePath)
+}
\ No newline at end of file
diff --git a/gradle.properties b/docs/pulsar-io-gen.sh
old mode 100644
new mode 100755
similarity index 59%
copy from gradle.properties
copy to docs/pulsar-io-gen.sh
index 20a29b3..0f2416a
--- a/gradle.properties
+++ b/docs/pulsar-io-gen.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,7 +18,17 @@
# under the License.
#
-org.gradle.configuration-cache=true
-org.gradle.parallel=true
-org.gradle.caching=true
-org.gradle.jvmargs=-Xmx4g -Xss2m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+# Check for the java to use
+if [[ -z $JAVA_HOME ]]; then
+ JAVA=$(which java)
+ if [ $? != 0 ]; then
+ echo "Error: JAVA_HOME not set, and no java executable found in
$PATH." 1>&2
+ exit 1
+ fi
+else
+ JAVA=$JAVA_HOME/bin/java
+fi
+CLASSPATH_FILE="$SCRIPT_DIR/build/classpath.txt"
+$SCRIPT_DIR/../gradlew exportClasspath --console=plain > /dev/null
+$JAVA -cp "$(cat $CLASSPATH_FILE)"
org.apache.pulsar.io.docs.ConnectorDocGenerator "$@"
\ No newline at end of file
diff --git
a/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java
b/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java
index 2e9d6a9..e159c88 100644
--- a/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java
+++ b/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java
@@ -28,6 +28,7 @@ import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
@@ -51,8 +52,11 @@ public class ConnectorDocGenerator implements
Callable<Integer> {
private static Reflections newReflections() throws Exception {
final String[] classpathList =
System.getProperty("java.class.path").split(":");
final List<URL> urlList = new ArrayList<>();
- for (String file : classpathList) {
- urlList.add(new File(file).toURI().toURL());
+ for (String cp : classpathList) {
+ File file = new File(cp);
+ if (file.isFile() && cp.endsWith(".nar")) {
+ urlList.add(new URL("jar:" + file.toURI() + "!/"));
+ }
}
return new Reflections(new ConfigurationBuilder().setUrls(urlList));
}
@@ -70,7 +74,7 @@ public class ConnectorDocGenerator implements
Callable<Integer> {
Field[] fields = configClass.getDeclaredFields();
for (Field field : fields) {
- if (Modifier.isStatic(field.getModifiers())) {
+ if (Modifier.isStatic(field.getModifiers()) ||
Modifier.isTransient(field.getModifiers())) {
continue;
}
FieldDoc fieldDoc = field.getDeclaredAnnotation(FieldDoc.class);
@@ -107,12 +111,16 @@ public class ConnectorDocGenerator implements
Callable<Integer> {
Set<Class<?>> connectorClasses =
reflections.getTypesAnnotatedWith(Connector.class);
log.info("Retrieve all `Connector` annotated classes : {}",
connectorClasses);
+ Path outputDirPath = Path.of(outputDir);
+ if (!Files.exists(outputDirPath)) {
+ Files.createDirectories(outputDirPath);
+ }
for (Class<?> connectorClass : connectorClasses) {
final Connector connectorDef =
connectorClass.getDeclaredAnnotation(Connector.class);
final String name = connectorDef.name().toLowerCase();
final String type = connectorDef.type().name().toLowerCase();
final String filename = "pulsar-io-%s-%s.yml".formatted(name,
type);
- final Path outputPath = Path.of(outputDir, filename);
+ final Path outputPath = outputDirPath.resolve(filename);
try (FileOutputStream fos = new
FileOutputStream(outputPath.toFile())) {
PrintWriter pw = new PrintWriter(new OutputStreamWriter(fos,
StandardCharsets.UTF_8));
generateConnectorYamlFile(connectorClass, connectorDef, pw);
@@ -125,7 +133,7 @@ public class ConnectorDocGenerator implements
Callable<Integer> {
names = {"-o", "--output-dir"},
description = "The output dir to dump connector docs",
required = true)
- String outputDir = null;
+ String outputDir;
@Option(names = {"-h", "--help"}, usageHelp = true, description = "Show
this help message")
boolean help = false;
diff --git a/dynamodb/build.gradle.kts b/dynamodb/build.gradle.kts
index 7697d16..980e18c 100644
--- a/dynamodb/build.gradle.kts
+++ b/dynamodb/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.common)
diff --git a/elastic-search/build.gradle.kts b/elastic-search/build.gradle.kts
index 620cb77..1b2022c 100644
--- a/elastic-search/build.gradle.kts
+++ b/elastic-search/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
compileOnly(libs.pulsar.io.core)
diff --git a/file/build.gradle.kts b/file/build.gradle.kts
index a22c7ac..3725b4f 100644
--- a/file/build.gradle.kts
+++ b/file/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/gradle.properties b/gradle.properties
index 20a29b3..fc83eee 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -17,7 +17,11 @@
# under the License.
#
+group=org.apache.pulsar
+version=4.3.0-SNAPSHOT
+
org.gradle.configuration-cache=true
+org.gradle.configureondemand=true
org.gradle.parallel=true
org.gradle.caching=true
org.gradle.jvmargs=-Xmx4g -Xss2m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 0e87504..743d543 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -16,26 +16,21 @@
# specific language governing permissions and limitations
# under the License.
#
-
[versions]
# Core
-pulsar = "4.1.3"
-pulsar-connectors = "4.3.0-SNAPSHOT"
+pulsar = "4.2.0"
java = "17"
-
-# Docker
-docker-jdk = "21"
-pulsar-client-python = "3.10.0"
-
# Code quality
-checkstyle = "10.14.2"
-
+checkstyle = "13.3.0"
+spotless = "8.4.0"
+idea-ext = "1.4.1"
# Major frameworks
bookkeeper = "4.17.3"
zookeeper = "3.9.5"
netty = "4.1.132.Final"
netty-iouring = "0.0.26.Final"
jetty = "12.1.5"
+jetty9 = "9.4.58.v20250814"
jersey = "2.42"
jackson = "2.18.6"
protobuf = "3.25.5"
@@ -43,16 +38,14 @@ grpc = "1.75.0"
slf4j = "2.0.17"
log4j2 = "2.25.3"
lombok = "1.18.42"
-
# OpenTelemetry
opentelemetry = "1.56.0"
opentelemetry-alpha = "1.56.0-alpha"
opentelemetry-instrumentation = "2.21.0"
opentelemetry-instrumentation-alpha = "2.21.0-alpha"
opentelemetry-semconv = "1.37.0"
-
# Apache Commons
-commons-lang3 = "3.19.0"
+commons-lang3 = "3.20.0"
commons-io = "2.21.0"
commons-codec = "1.20.0"
commons-compress = "1.28.0"
@@ -63,7 +56,6 @@ commons-math3 = "3.6.1"
commons-logging = "1.3.5"
commons-beanutils = "1.11.0"
commons-configuration2 = "2.12.0"
-
# BouncyCastle
bouncycastle-bcprov = "1.78.1"
bouncycastle-bcpkix = "1.81"
@@ -71,49 +63,40 @@ bouncycastle-bcutil = "1.81"
bouncycastle-bcprov-ext = "1.78.1"
bouncycastle-bcpkix-fips = "2.0.10"
bouncycastle-bc-fips = "2.0.1"
-
# Serialization
avro = "1.12.0"
gson = "2.13.2"
snakeyaml = "2.0"
-
# Vert.x
vertx = "4.5.24"
-
# Networking / HTTP
asynchttpclient = "2.12.4"
conscrypt = "2.5.2"
okhttp3 = "5.3.1"
okio = "3.16.3"
-netty-tcnative = "2.0.75.Final"
httpcomponents-httpclient = "4.5.13"
httpcomponents-httpcore = "4.4.15"
-
# Google libraries (transitive deps, versions managed to match Maven)
google-auth = "1.24.1"
google-http-client = "1.41.0"
j2objc-annotations = "1.3"
opencensus = "0.28.0"
opentelemetry-gcp-resources = "1.48.0-alpha"
-
# Data structures / Utils
guava = "33.4.8-jre"
caffeine = "3.2.3"
-fastutil = "8.5.16"
jctools = "4.0.5"
roaringbitmap = "1.6.9"
hppc = "0.9.1"
aircompressor = "2.0.3"
completable-futures = "0.3.6"
re2j = "1.8"
-
# Metrics / Observability
prometheus = "0.16.0"
prometheus-jmx = "0.16.1"
dropwizardmetrics = "4.1.12.1"
hdrHistogram = "2.1.9"
perfmark = "0.26.0"
-
# Auth
jsonwebtoken = "0.13.0"
athenz = "1.10.62"
@@ -121,18 +104,15 @@ jose4j = "0.9.6"
nimbus-jose-jwt = "9.37.4"
auth0-java-jwt = "4.3.0"
auth0-jwks-rsa = "0.22.0"
-
# CLI
picocli = "4.7.5"
jline3 = "3.21.0"
jline2 = "2.14.6"
-
javassist = "3.25.0-GA"
rocksdb = "7.9.2"
kotlin-stdlib = "1.8.20"
audience-annotations = "0.12.0"
jetbrains-annotations = "13.0"
-
# Misc
curator = "5.7.1"
reflections = "0.10.2"
@@ -151,7 +131,6 @@ guice = "5.1.0"
snappy = "1.1.10.8"
ipaddress = "5.5.0"
zt-zip = "1.17"
-
# Jakarta
jakarta-ws-rs = "2.1.6"
jakarta-annotation = "1.3.5"
@@ -159,17 +138,14 @@ jakarta-activation = "1.2.2"
jakarta-xml-bind = "2.3.3"
jakarta-validation = "2.0.2"
javax-servlet = "3.1.0"
-
# Oxia / etcd
oxia = "0.7.4"
-
# Build plugins
-lightproto = "0.6.2"
+lightproto = "0.6.6"
errorprone = "2.45.0"
spotbugs = "4.9.6"
checkerframework = "3.33.0"
jsr305 = "3.0.2"
-
# Test
testng = "7.7.1"
mockito = "5.19.0"
@@ -189,16 +165,13 @@ skyscreamer = "1.5.0"
zstd-jni = "1.5.7-3"
lz4java = "1.10.3"
spring = "6.2.12"
-
# Connectors / IO
kafka-client = "4.1.1"
confluent = "8.1.1"
opensearch = "2.19.4"
elasticsearch-java = "8.15.3"
debezium = "3.4.2.Final"
-debezium-mysql-connector = "9.4.0"
kubernetesclient = "23.0.0"
-
# IO connector specific
aerospike-client = "4.5.0"
aws-sdk = "1.12.788"
@@ -214,67 +187,47 @@ solr = "9.8.0"
hbase = "2.6.4-hadoop3"
hadoop3 = "3.4.3"
jclouds = "2.6.0"
-
+openmldb = "0.4.4-hotfix1"
+docker-java = "3.7.1"
# Shading
-shadow = "9.3.2"
+shadow = "9.4.1"
[libraries]
# SLF4J
-slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
-jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j", version.ref = "slf4j" }
-
+slf4j-bom = { module = "org.slf4j:slf4j-bom", version.ref = "slf4j" }
+slf4j-api = { module = "org.slf4j:slf4j-api" }
+jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j" }
# Log4j2
-log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref =
"log4j2" }
-log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref =
"log4j2" }
-log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref =
"log4j2" }
-log4j-layout-template-json = { module =
"org.apache.logging.log4j:log4j-layout-template-json", version.ref = "log4j2" }
-log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl",
version.ref = "log4j2" }
-
+log4j-bom = { module = "org.apache.logging.log4j:log4j-bom", version.ref =
"log4j2" }
+log4j-api = { module = "org.apache.logging.log4j:log4j-api" }
+log4j-core = { module = "org.apache.logging.log4j:log4j-core" }
+log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl" }
# Lombok
lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" }
-
# Jackson
jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref =
"jackson" }
-jackson-annotations = { module =
"com.fasterxml.jackson.core:jackson-annotations", version.ref = "jackson" }
-jackson-core = { module = "com.fasterxml.jackson.core:jackson-core",
version.ref = "jackson" }
-jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind",
version.ref = "jackson" }
-jackson-module-parameter-names = { module =
"com.fasterxml.jackson.module:jackson-module-parameter-names", version.ref =
"jackson" }
-jackson-datatype-jsr310 = { module =
"com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref =
"jackson" }
-jackson-datatype-jdk8 = { module =
"com.fasterxml.jackson.datatype:jackson-datatype-jdk8", version.ref = "jackson"
}
-jackson-dataformat-yaml = { module =
"com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version.ref =
"jackson" }
-jackson-module-jsonSchema = { module =
"com.fasterxml.jackson.module:jackson-module-jsonSchema", version.ref =
"jackson" }
-jackson-jaxrs-json-provider = { module =
"com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider", version.ref =
"jackson" }
-
+jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind" }
+jackson-datatype-jsr310 = { module =
"com.fasterxml.jackson.datatype:jackson-datatype-jsr310" }
+jackson-dataformat-yaml = { module =
"com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" }
+jackson-dataformat-cbor = { module =
"com.fasterxml.jackson.dataformat:jackson-dataformat-cbor" }
# Netty
netty-bom = { module = "io.netty:netty-bom", version.ref = "netty" }
-netty-common = { module = "io.netty:netty-common", version.ref = "netty" }
-netty-buffer = { module = "io.netty:netty-buffer", version.ref = "netty" }
-netty-handler = { module = "io.netty:netty-handler", version.ref = "netty" }
-netty-transport = { module = "io.netty:netty-transport", version.ref = "netty"
}
-netty-codec-http = { module = "io.netty:netty-codec-http", version.ref =
"netty" }
-netty-codec-http2 = { module = "io.netty:netty-codec-http2", version.ref =
"netty" }
-netty-codec-haproxy = { module = "io.netty:netty-codec-haproxy", version.ref =
"netty" }
-netty-handler-proxy = { module = "io.netty:netty-handler-proxy", version.ref =
"netty" }
-netty-codec-socks = { module = "io.netty:netty-codec-socks", version.ref =
"netty" }
-netty-resolver-dns = { module = "io.netty:netty-resolver-dns", version.ref =
"netty" }
-netty-resolver-dns-native-macos = { module =
"io.netty:netty-resolver-dns-native-macos", version.ref = "netty" }
-netty-transport-native-epoll = { module =
"io.netty:netty-transport-native-epoll", version.ref = "netty" }
-netty-transport-native-unix-common = { module =
"io.netty:netty-transport-native-unix-common", version.ref = "netty" }
-netty-tcnative-boringssl-static = { module =
"io.netty:netty-tcnative-boringssl-static", version.ref = "netty-tcnative" }
+netty-buffer = { module = "io.netty:netty-buffer" }
+netty-handler = { module = "io.netty:netty-handler" }
+netty-codec-http = { module = "io.netty:netty-codec-http" }
netty-incubator-transport-classes-io_uring = { module =
"io.netty.incubator:netty-incubator-transport-classes-io_uring", version.ref =
"netty-iouring" }
netty-incubator-transport-native-io-uring = { module =
"io.netty.incubator:netty-incubator-transport-native-io_uring", version.ref =
"netty-iouring" }
netty-reactive-streams = { module =
"com.typesafe.netty:netty-reactive-streams", version.ref =
"netty-reactive-streams" }
-
# Protobuf / gRPC
-protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref =
"protobuf" }
-protobuf-java-util = { module = "com.google.protobuf:protobuf-java-util",
version.ref = "protobuf" }
-grpc-all = { module = "io.grpc:grpc-all", version.ref = "grpc" }
-grpc-netty-shaded = { module = "io.grpc:grpc-netty-shaded", version.ref =
"grpc" }
-grpc-stub = { module = "io.grpc:grpc-stub", version.ref = "grpc" }
-
+protobuf-bom = { module = "com.google.protobuf:protobuf-bom", version.ref =
"protobuf" }
+protobuf-java = { module = "com.google.protobuf:protobuf-java" }
+protobuf-java-util = { module = "com.google.protobuf:protobuf-java-util" }
+grpc-bom = { module = "io.grpc:grpc-bom", version.ref = "grpc" }
+grpc-all = { module = "io.grpc:grpc-all" }
+grpc-netty-shaded = { module = "io.grpc:grpc-netty-shaded" }
+grpc-stub = { module = "io.grpc:grpc-stub" }
# Guava
guava = { module = "com.google.guava:guava", version.ref = "guava" }
-
# Apache Commons
commons-lang3 = { module = "org.apache.commons:commons-lang3", version.ref =
"commons-lang3" }
commons-io = { module = "commons-io:commons-io", version.ref = "commons-io" }
@@ -283,7 +236,6 @@ commons-compress = { module =
"org.apache.commons:commons-compress", version.ref
commons-collections4 = { module = "org.apache.commons:commons-collections4",
version.ref = "commons-collections4" }
commons-cli = { module = "commons-cli:commons-cli", version.ref =
"commons-cli" }
commons-math3 = { module = "org.apache.commons:commons-math3", version.ref =
"commons-math3" }
-
# BookKeeper
bookkeeper-server = { module = "org.apache.bookkeeper:bookkeeper-server",
version.ref = "bookkeeper" }
bookkeeper-common-allocator = { module =
"org.apache.bookkeeper:bookkeeper-common-allocator", version.ref = "bookkeeper"
}
@@ -297,86 +249,74 @@ bookkeeper-stream-storage-service-impl = { module =
"org.apache.bookkeeper:strea
bookkeeper-tools-framework = { module =
"org.apache.bookkeeper:bookkeeper-tools-framework", version.ref = "bookkeeper" }
bookkeeper-http-vertx-server = { module =
"org.apache.bookkeeper.http:vertx-http-server", version.ref = "bookkeeper" }
distributedlog-core = { module =
"org.apache.distributedlog:distributedlog-core", version.ref = "bookkeeper" }
-
# ZooKeeper
zookeeper = { module = "org.apache.zookeeper:zookeeper", version.ref =
"zookeeper" }
zookeeper-tests = { module = "org.apache.zookeeper:zookeeper", version.ref =
"zookeeper" }
-
# Curator
curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref
= "curator" }
-
# Conscrypt
conscrypt-openjdk-uber = { module = "org.conscrypt:conscrypt-openjdk-uber",
version.ref = "conscrypt" }
-
# Jetty
-jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref =
"jetty" }
-jetty-util = { module = "org.eclipse.jetty:jetty-util", version.ref = "jetty" }
-jetty-alpn-conscrypt-server = { module =
"org.eclipse.jetty:jetty-alpn-conscrypt-server", version.ref = "jetty" }
-jetty-compression-server = { module =
"org.eclipse.jetty.compression:jetty-compression-server", version.ref = "jetty"
}
-jetty-compression-gzip = { module =
"org.eclipse.jetty.compression:jetty-compression-gzip", version.ref = "jetty" }
-jetty-client = { module = "org.eclipse.jetty:jetty-client", version.ref =
"jetty" }
-jetty-ee8-servlet = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlet",
version.ref = "jetty" }
-jetty-ee8-servlets = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlets",
version.ref = "jetty" }
-jetty-ee8-proxy = { module = "org.eclipse.jetty.ee8:jetty-ee8-proxy",
version.ref = "jetty" }
-jetty-websocket-jetty-api = { module =
"org.eclipse.jetty.websocket:jetty-websocket-jetty-api", version.ref = "jetty" }
-jetty-websocket-jetty-client = { module =
"org.eclipse.jetty.websocket:jetty-websocket-jetty-client", version.ref =
"jetty" }
-jetty-ee8-websocket-jetty-server = { module =
"org.eclipse.jetty.ee8.websocket:jetty-ee8-websocket-jetty-server", version.ref
= "jetty" }
-
+jetty-bom = { module = "org.eclipse.jetty:jetty-bom", version.ref = "jetty" }
+jetty9-bom = { module = "org.eclipse.jetty:jetty-bom", version.ref = "jetty9" }
+jetty-server = { module = "org.eclipse.jetty:jetty-server" }
+jetty-util = { module = "org.eclipse.jetty:jetty-util" }
+jetty-alpn-conscrypt-server = { module =
"org.eclipse.jetty:jetty-alpn-conscrypt-server" }
+jetty-compression-server = { module =
"org.eclipse.jetty.compression:jetty-compression-server" }
+jetty-compression-gzip = { module =
"org.eclipse.jetty.compression:jetty-compression-gzip" }
+jetty-client = { module = "org.eclipse.jetty:jetty-client" }
+jetty-ee8-servlet = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlet" }
+jetty-ee8-servlets = { module = "org.eclipse.jetty.ee8:jetty-ee8-servlets" }
+jetty-ee8-proxy = { module = "org.eclipse.jetty.ee8:jetty-ee8-proxy" }
+jetty-websocket-jetty-api = { module =
"org.eclipse.jetty.websocket:jetty-websocket-jetty-api" }
+jetty-websocket-jetty-client = { module =
"org.eclipse.jetty.websocket:jetty-websocket-jetty-client" }
+jetty-ee8-websocket-jetty-server = { module =
"org.eclipse.jetty.ee8.websocket:jetty-ee8-websocket-jetty-server" }
# Jersey
-jersey-server = { module = "org.glassfish.jersey.core:jersey-server",
version.ref = "jersey" }
-jersey-container-servlet-core = { module =
"org.glassfish.jersey.containers:jersey-container-servlet-core", version.ref =
"jersey" }
-jersey-container-servlet = { module =
"org.glassfish.jersey.containers:jersey-container-servlet", version.ref =
"jersey" }
-jersey-media-json-jackson = { module =
"org.glassfish.jersey.media:jersey-media-json-jackson", version.ref = "jersey" }
-jersey-client = { module = "org.glassfish.jersey.core:jersey-client",
version.ref = "jersey" }
-jersey-hk2 = { module = "org.glassfish.jersey.inject:jersey-hk2", version.ref
= "jersey" }
-jersey-media-multipart = { module =
"org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" }
-jersey-test-framework-core = { module =
"org.glassfish.jersey.test-framework:jersey-test-framework-core", version.ref =
"jersey" }
-jersey-test-framework-grizzly2 = { module =
"org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2",
version.ref = "jersey" }
-
+jersey-bom = { module = "org.glassfish.jersey:jersey-bom", version.ref =
"jersey" }
+jersey-server = { module = "org.glassfish.jersey.core:jersey-server" }
+jersey-container-servlet-core = { module =
"org.glassfish.jersey.containers:jersey-container-servlet-core" }
+jersey-container-servlet = { module =
"org.glassfish.jersey.containers:jersey-container-servlet" }
+jersey-media-json-jackson = { module =
"org.glassfish.jersey.media:jersey-media-json-jackson" }
+jersey-client = { module = "org.glassfish.jersey.core:jersey-client" }
+jersey-hk2 = { module = "org.glassfish.jersey.inject:jersey-hk2" }
+jersey-media-multipart = { module =
"org.glassfish.jersey.media:jersey-media-multipart" }
+jersey-test-framework-core = { module =
"org.glassfish.jersey.test-framework:jersey-test-framework-core" }
+jersey-test-framework-grizzly2 = { module =
"org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2"
}
# Prometheus
-simpleclient = { module = "io.prometheus:simpleclient", version.ref =
"prometheus" }
-simpleclient-hotspot = { module = "io.prometheus:simpleclient_hotspot",
version.ref = "prometheus" }
-simpleclient-caffeine = { module = "io.prometheus:simpleclient_caffeine",
version.ref = "prometheus" }
-simpleclient-httpserver = { module = "io.prometheus:simpleclient_httpserver",
version.ref = "prometheus" }
-simpleclient-servlet = { module = "io.prometheus:simpleclient_servlet",
version.ref = "prometheus" }
-simpleclient-common = { module = "io.prometheus:simpleclient_common",
version.ref = "prometheus" }
-simpleclient-log4j2 = { module = "io.prometheus:simpleclient_log4j2",
version.ref = "prometheus" }
+simpleclient-bom = { module = "io.prometheus:simpleclient_bom", version.ref =
"prometheus" }
prometheus-jmx-collector = { module = "io.prometheus.jmx:collector",
version.ref = "prometheus-jmx" }
-
# OpenTelemetry
-opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api",
version.ref = "opentelemetry" }
-opentelemetry-api-incubator = { module =
"io.opentelemetry:opentelemetry-api-incubator", version.ref =
"opentelemetry-alpha" }
-opentelemetry-sdk = { module = "io.opentelemetry:opentelemetry-sdk",
version.ref = "opentelemetry" }
-opentelemetry-sdk-extension-autoconfigure = { module =
"io.opentelemetry:opentelemetry-sdk-extension-autoconfigure", version.ref =
"opentelemetry" }
-opentelemetry-sdk-testing = { module =
"io.opentelemetry:opentelemetry-sdk-testing", version.ref = "opentelemetry" }
-opentelemetry-exporter-otlp = { module =
"io.opentelemetry:opentelemetry-exporter-otlp", version.ref = "opentelemetry" }
-opentelemetry-exporter-prometheus = { module =
"io.opentelemetry:opentelemetry-exporter-prometheus", version.ref =
"opentelemetry-alpha" }
-opentelemetry-instrumentation-resources = { module =
"io.opentelemetry.instrumentation:opentelemetry-resources", version.ref =
"opentelemetry-instrumentation-alpha" }
-opentelemetry-instrumentation-runtime-telemetry-java17 = { module =
"io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17",
version.ref = "opentelemetry-instrumentation-alpha" }
+opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom",
version.ref = "opentelemetry" }
+opentelemetry-bom-alpha = { module =
"io.opentelemetry:opentelemetry-bom-alpha", version.ref = "opentelemetry-alpha"
}
+opentelemetry-instrumentation-bom = { module =
"io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom",
version.ref = "opentelemetry-instrumentation" }
+opentelemetry-instrumentation-bom-alpha = { module =
"io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha",
version.ref = "opentelemetry-instrumentation-alpha" }
+opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api" }
+opentelemetry-api-incubator = { module =
"io.opentelemetry:opentelemetry-api-incubator" }
+opentelemetry-sdk = { module = "io.opentelemetry:opentelemetry-sdk" }
+opentelemetry-sdk-extension-autoconfigure = { module =
"io.opentelemetry:opentelemetry-sdk-extension-autoconfigure" }
+opentelemetry-sdk-testing = { module =
"io.opentelemetry:opentelemetry-sdk-testing" }
+opentelemetry-exporter-otlp = { module =
"io.opentelemetry:opentelemetry-exporter-otlp" }
+opentelemetry-exporter-prometheus = { module =
"io.opentelemetry:opentelemetry-exporter-prometheus" }
+opentelemetry-instrumentation-resources = { module =
"io.opentelemetry.instrumentation:opentelemetry-resources" }
+opentelemetry-instrumentation-runtime-telemetry-java17 = { module =
"io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17" }
opentelemetry-semconv = { module =
"io.opentelemetry.semconv:opentelemetry-semconv", version.ref =
"opentelemetry-semconv" }
opentelemetry-gcp-resources = { module =
"io.opentelemetry.contrib:opentelemetry-gcp-resources", version.ref =
"opentelemetry-gcp-resources" }
-
# BouncyCastle
bcpkix-jdk18on = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref =
"bouncycastle-bcpkix" }
bcprov-ext-jdk18on = { module = "org.bouncycastle:bcprov-ext-jdk18on",
version.ref = "bouncycastle-bcprov-ext" }
bcpkix-fips = { module = "org.bouncycastle:bcpkix-fips", version.ref =
"bouncycastle-bcpkix-fips" }
bc-fips = { module = "org.bouncycastle:bc-fips", version.ref =
"bouncycastle-bc-fips" }
-bcutil-fips = { module = "org.bouncycastle:bcutil-fips", version = "2.0.5" }
-
+bcutil-fips = "org.bouncycastle:bcutil-fips:2.0.5"
# RocksDB
rocksdbjni = { module = "org.rocksdb:rocksdbjni", version.ref = "rocksdb" }
-
# Error Prone
error-prone-annotations = { module =
"com.google.errorprone:error_prone_annotations", version.ref = "errorprone" }
-
# Data structures
caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref =
"caffeine" }
-fastutil = { module = "it.unimi.dsi:fastutil", version.ref = "fastutil" }
jctools-core = { module = "org.jctools:jctools-core", version.ref = "jctools" }
roaringbitmap = { module = "org.roaringbitmap:RoaringBitmap", version.ref =
"roaringbitmap" }
hppc = { module = "com.carrotsearch:hppc", version.ref = "hppc" }
aircompressor = { module = "io.airlift:aircompressor", version.ref =
"aircompressor" }
-
# Misc libs
gson = { module = "com.google.code.gson:gson", version.ref = "gson" }
re2j = { module = "com.google.re2j:re2j", version.ref = "re2j" }
@@ -391,10 +331,12 @@ javassist = { module = "org.javassist:javassist",
version.ref = "javassist" }
commons-text = { module = "org.apache.commons:commons-text", version.ref =
"commons-text" }
typetools = { module = "net.jodah:typetools", version.ref = "typetools" }
perfmark-api = { module = "io.perfmark:perfmark-api", version.ref = "perfmark"
}
-okhttp3 = { module = "com.squareup.okhttp3:okhttp-jvm", version.ref =
"okhttp3" }
-okhttp3-logging-interceptor = { module =
"com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp3" }
-okio = { module = "com.squareup.okio:okio", version.ref = "okio" }
-# Transitive dep version pins (enforced by pulsar-dependencies platform)
+okhttp3-bom = { module = "com.squareup.okhttp3:okhttp-bom", version.ref =
"okhttp3" }
+okhttp3 = { module = "com.squareup.okhttp3:okhttp-jvm" }
+okhttp3-logging-interceptor = { module =
"com.squareup.okhttp3:logging-interceptor" }
+okio-bom = { module = "com.squareup.okio:okio-bom", version.ref = "okio" }
+okio = { module = "com.squareup.okio:okio" }
+# Transitive dep version pins (enforced by pulsar-connectors-dependencies
platform)
google-auth-library-credentials = { module =
"com.google.auth:google-auth-library-credentials", version.ref = "google-auth" }
google-auth-library-oauth2-http = { module =
"com.google.auth:google-auth-library-oauth2-http", version.ref = "google-auth" }
google-http-client = { module = "com.google.http-client:google-http-client",
version.ref = "google-http-client" }
@@ -406,7 +348,6 @@ httpcomponents-httpclient = { module =
"org.apache.httpcomponents:httpclient", v
httpcomponents-httpcore = { module = "org.apache.httpcomponents:httpcore",
version.ref = "httpcomponents-httpcore" }
jakarta-annotation-api = { module =
"jakarta.annotation:jakarta.annotation-api", version.ref = "jakarta-annotation"
}
kotlin-stdlib = { module = "org.jetbrains.kotlin:kotlin-stdlib", version.ref =
"kotlin-stdlib" }
-
snakeyaml = { module = "org.yaml:snakeyaml", version.ref = "snakeyaml" }
ant = { module = "org.apache.ant:ant", version.ref = "ant" }
guice = { module = "com.google.inject:guice", version.ref = "guice" }
@@ -416,7 +357,7 @@ vertx-core = { module = "io.vertx:vertx-core", version.ref
= "vertx" }
vertx-web = { module = "io.vertx:vertx-web", version.ref = "vertx" }
avro = { module = "org.apache.avro:avro", version.ref = "avro" }
avro-protobuf = { module = "org.apache.avro:avro-protobuf", version.ref =
"avro" }
-joda-time = { module = "joda-time:joda-time", version = "2.10.10" }
+joda-time = "joda-time:joda-time:2.10.10"
sketches-core = { module = "com.yahoo.datasketches:sketches-core", version.ref
= "sketches" }
java-semver = { module = "com.github.zafarkhaja:java-semver", version.ref =
"java-semver" }
oshi-core = { module = "com.github.oshi:oshi-core-java11", version.ref =
"oshi" }
@@ -426,16 +367,15 @@ dropwizardmetrics-core = { module =
"io.dropwizard.metrics:metrics-core", versio
dropwizardmetrics-graphite = { module =
"io.dropwizard.metrics:metrics-graphite", version.ref = "dropwizardmetrics" }
dropwizardmetrics-jvm = { module = "io.dropwizard.metrics:metrics-jvm",
version.ref = "dropwizardmetrics" }
snappy-java = { module = "org.xerial.snappy:snappy-java", version.ref =
"snappy" }
-jspecify = { module = "org.jspecify:jspecify", version = "1.0.0" }
+jspecify = "org.jspecify:jspecify:1.0.0"
reflections = { module = "org.reflections:reflections", version.ref =
"reflections" }
-
# Auth / Security
-jjwt-api = { module = "io.jsonwebtoken:jjwt-api", version.ref = "jsonwebtoken"
}
-jjwt-impl = { module = "io.jsonwebtoken:jjwt-impl", version.ref =
"jsonwebtoken" }
-jjwt-jackson = { module = "io.jsonwebtoken:jjwt-jackson", version.ref =
"jsonwebtoken" }
+jjwt-bom = { module = "io.jsonwebtoken:jjwt-bom", version.ref = "jsonwebtoken"
}
+jjwt-api = { module = "io.jsonwebtoken:jjwt-api" }
+jjwt-impl = { module = "io.jsonwebtoken:jjwt-impl" }
+jjwt-jackson = { module = "io.jsonwebtoken:jjwt-jackson" }
auth0-java-jwt = { module = "com.auth0:java-jwt", version.ref =
"auth0-java-jwt" }
auth0-jwks-rsa = { module = "com.auth0:jwks-rsa", version.ref =
"auth0-jwks-rsa" }
-
# Jakarta
jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref
= "jakarta-ws-rs" }
jakarta-activation-api = { module =
"jakarta.activation:jakarta.activation-api", version.ref = "jakarta-activation"
}
@@ -444,18 +384,16 @@ jakarta-xml-bind-api = { module =
"jakarta.xml.bind:jakarta.xml.bind-api", versi
javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version.ref
= "javax-servlet" }
zt-zip = { module = "org.zeroturnaround:zt-zip", version.ref = "zt-zip" }
ipaddress = { module = "com.github.seancfoley:ipaddress", version.ref =
"ipaddress" }
-
# Oxia / etcd
oxia-client = { module = "io.github.oxia-db:oxia-client", version.ref = "oxia"
}
oxia-testcontainers = { module = "io.github.oxia-db:oxia-testcontainers",
version.ref = "oxia" }
-
# Static analysis
spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations",
version.ref = "spotbugs" }
jsr305 = { module = "com.google.code.findbugs:jsr305", version.ref = "jsr305" }
-
# Test
testng = { module = "org.testng:testng", version.ref = "testng" }
-mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" }
+mockito-bom = { module = "org.mockito:mockito-bom", version.ref = "mockito" }
+mockito-core = { module = "org.mockito:mockito-core" }
awaitility = { module = "org.awaitility:awaitility", version.ref =
"awaitility" }
assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" }
hamcrest = { module = "org.hamcrest:hamcrest", version.ref = "hamcrest" }
@@ -475,33 +413,35 @@ spring-jdbc = { module =
"org.springframework:spring-jdbc", version.ref = "sprin
spring-orm = { module = "org.springframework:spring-orm", version.ref =
"spring" }
kubernetes-client-java = { module = "io.kubernetes:client-java", version.ref =
"kubernetesclient" }
kubernetes-client-java-api-fluent = { module =
"io.kubernetes:client-java-api-fluent", version.ref = "kubernetesclient" }
-testcontainers = { module = "org.testcontainers:testcontainers", version.ref =
"testcontainers" }
-testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch",
version.ref = "testcontainers" }
-testcontainers-toxiproxy = { module = "org.testcontainers:toxiproxy",
version.ref = "testcontainers" }
-testcontainers-localstack = { module = "org.testcontainers:localstack",
version.ref = "testcontainers" }
-testcontainers-rabbitmq = { module = "org.testcontainers:rabbitmq",
version.ref = "testcontainers" }
-testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref =
"testcontainers" }
-testcontainers-mysql = { module = "org.testcontainers:mysql", version.ref =
"testcontainers" }
-testcontainers-postgresql = { module = "org.testcontainers:postgresql",
version.ref = "testcontainers" }
-testcontainers-mongodb = { module = "org.testcontainers:mongodb", version.ref
= "testcontainers" }
-testcontainers-pulsar = { module = "org.testcontainers:pulsar", version.ref =
"testcontainers" }
-testcontainers-cassandra = { module = "org.testcontainers:cassandra",
version.ref = "testcontainers" }
-testcontainers-k3s = { module = "org.testcontainers:k3s", version.ref =
"testcontainers" }
+docker-java-bom = { module = "com.github.docker-java:docker-java-bom",
version.ref = "docker-java" }
+testcontainers-bom = { module = "org.testcontainers:testcontainers-bom",
version.ref = "testcontainers" }
+testcontainers = { module = "org.testcontainers:testcontainers" }
+testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch" }
+testcontainers-toxiproxy = { module = "org.testcontainers:toxiproxy" }
+testcontainers-localstack = { module = "org.testcontainers:localstack" }
+testcontainers-rabbitmq = { module = "org.testcontainers:rabbitmq" }
+testcontainers-kafka = { module = "org.testcontainers:kafka" }
+testcontainers-mysql = { module = "org.testcontainers:mysql" }
+testcontainers-postgresql = { module = "org.testcontainers:postgresql" }
+testcontainers-mongodb = { module = "org.testcontainers:mongodb" }
+testcontainers-pulsar = { module = "org.testcontainers:pulsar" }
+testcontainers-cassandra = { module = "org.testcontainers:cassandra" }
+testcontainers-k3s = { module = "org.testcontainers:k3s" }
kerby-simplekdc = { module = "org.apache.kerby:kerb-simplekdc", version.ref =
"kerby" }
json = { module = "org.json:json", version.ref = "json" }
-
# AWS SDKs
-aws-java-sdk-core = { module = "com.amazonaws:aws-java-sdk-core", version.ref
= "aws-sdk" }
-aws-java-sdk-sts = { module = "com.amazonaws:aws-java-sdk-sts", version.ref =
"aws-sdk" }
-aws-sdk2-regions = { module = "software.amazon.awssdk:regions", version.ref =
"aws-sdk2" }
-aws-sdk2-sts = { module = "software.amazon.awssdk:sts", version.ref =
"aws-sdk2" }
-aws-sdk2-kinesis = { module = "software.amazon.awssdk:kinesis", version.ref =
"aws-sdk2" }
-aws-sdk2-utils = { module = "software.amazon.awssdk:utils", version.ref =
"aws-sdk2" }
-dynamodb-streams-kinesis-adapter = { module =
"com.amazonaws:dynamodb-streams-kinesis-adapter", version = "1.6.0" }
-amazon-kinesis-client = { module =
"software.amazon.kinesis:amazon-kinesis-client", version = "2.6.0" }
-amazon-kinesis-client-v3 = { module =
"software.amazon.kinesis:amazon-kinesis-client", version = "3.1.2" }
-amazon-kinesis-producer = { module =
"software.amazon.kinesis:amazon-kinesis-producer", version = "1.0.4" }
-
+aws-java-sdk-bom = { module = "com.amazonaws:aws-java-sdk-bom", version.ref =
"aws-sdk" }
+aws-java-sdk-core = { module = "com.amazonaws:aws-java-sdk-core" }
+aws-java-sdk-sts = { module = "com.amazonaws:aws-java-sdk-sts" }
+aws-sdk2-bom = { module = "software.amazon.awssdk:bom", version.ref =
"aws-sdk2" }
+aws-sdk2-regions = { module = "software.amazon.awssdk:regions" }
+aws-sdk2-sts = { module = "software.amazon.awssdk:sts" }
+aws-sdk2-kinesis = { module = "software.amazon.awssdk:kinesis" }
+aws-sdk2-utils = { module = "software.amazon.awssdk:utils" }
+dynamodb-streams-kinesis-adapter =
"com.amazonaws:dynamodb-streams-kinesis-adapter:1.6.0"
+amazon-kinesis-client = "software.amazon.kinesis:amazon-kinesis-client:2.6.0"
+amazon-kinesis-client-v3 =
"software.amazon.kinesis:amazon-kinesis-client:3.1.2"
+amazon-kinesis-producer =
"software.amazon.kinesis:amazon-kinesis-producer:1.0.4"
# Kafka / Confluent
kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref =
"kafka-client" }
kafka-connect-runtime = { module = "org.apache.kafka:connect-runtime",
version.ref = "kafka-client" }
@@ -512,104 +452,95 @@ kafka-connect-file = { module =
"org.apache.kafka:connect-file", version.ref = "
kafka-schema-registry-client = { module =
"io.confluent:kafka-schema-registry-client", version.ref = "confluent" }
kafka-avro-serializer = { module = "io.confluent:kafka-avro-serializer",
version.ref = "confluent" }
kafka-connect-avro-converter = { module =
"io.confluent:kafka-connect-avro-converter", version.ref = "confluent" }
-
# ElasticSearch / OpenSearch
opensearch-rest-high-level-client = { module =
"org.opensearch.client:opensearch-rest-high-level-client", version.ref =
"opensearch" }
elasticsearch-java = { module = "co.elastic.clients:elasticsearch-java",
version.ref = "elasticsearch-java" }
-
# Debezium
-debezium-core = { module = "io.debezium:debezium-core", version.ref =
"debezium" }
-debezium-connector-mysql = { module = "io.debezium:debezium-connector-mysql",
version.ref = "debezium" }
-debezium-connector-mongodb = { module =
"io.debezium:debezium-connector-mongodb", version.ref = "debezium" }
-debezium-connector-postgres = { module =
"io.debezium:debezium-connector-postgres", version.ref = "debezium" }
-debezium-connector-oracle = { module =
"io.debezium:debezium-connector-oracle", version.ref = "debezium" }
-debezium-connector-sqlserver = { module =
"io.debezium:debezium-connector-sqlserver", version.ref = "debezium" }
-
+debezium-bom = { module = "io.debezium:debezium-bom", version.ref = "debezium"
}
+debezium-core = { module = "io.debezium:debezium-core" }
+debezium-connector-mysql = { module = "io.debezium:debezium-connector-mysql" }
+debezium-connector-mongodb = { module =
"io.debezium:debezium-connector-mongodb" }
+debezium-connector-postgres = { module =
"io.debezium:debezium-connector-postgres" }
+debezium-connector-oracle = { module = "io.debezium:debezium-connector-oracle"
}
+debezium-connector-sqlserver = { module =
"io.debezium:debezium-connector-sqlserver" }
# Database drivers
-postgresql-jdbc = { module = "org.postgresql:postgresql", version = "42.7.10" }
-sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version = "3.47.1.0" }
-clickhouse-jdbc = { module = "com.clickhouse:clickhouse-jdbc", version =
"0.4.6" }
-mariadb-jdbc = { module = "org.mariadb.jdbc:mariadb-java-client", version =
"3.5.5" }
-openmldb-jdbc = { module = "com.4paradigm.openmldb:openmldb-jdbc", version =
"0.4.4-hotfix1" }
-openmldb-native = { module = "com.4paradigm.openmldb:openmldb-native", version
= "0.4.4-hotfix1" }
-
+postgresql-jdbc = "org.postgresql:postgresql:42.7.10"
+sqlite-jdbc = "org.xerial:sqlite-jdbc:3.47.1.0"
+clickhouse-jdbc = "com.clickhouse:clickhouse-jdbc:0.4.6"
+mariadb-jdbc = "org.mariadb.jdbc:mariadb-java-client:3.5.5"
+mysql-connector-j = "com.mysql:mysql-connector-j:9.4.0"
+openmldb-jdbc = { module = "com.4paradigm.openmldb:openmldb-jdbc", version.ref
= "openmldb" }
+openmldb-native = { module = "com.4paradigm.openmldb:openmldb-native",
version.ref = "openmldb" }
# JClouds
jclouds-allblobstore = { module = "org.apache.jclouds:jclouds-allblobstore",
version.ref = "jclouds" }
jclouds-blobstore = { module = "org.apache.jclouds:jclouds-blobstore",
version.ref = "jclouds" }
-
# Hadoop
hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref =
"hadoop3" }
hadoop-hdfs-client = { module = "org.apache.hadoop:hadoop-hdfs-client",
version.ref = "hadoop3" }
hadoop-client = { module = "org.apache.hadoop:hadoop-client", version.ref =
"hadoop3" }
hadoop-minicluster = { module = "org.apache.hadoop:hadoop-minicluster",
version.ref = "hadoop3" }
-
# HBase
hbase-client = { module = "org.apache.hbase:hbase-client", version.ref =
"hbase" }
hbase-common = { module = "org.apache.hbase:hbase-common", version.ref =
"hbase" }
-
# NoSQL / Search
aerospike-client = { module = "com.aerospike:aerospike-client-bc", version.ref
= "aerospike-client" }
cassandra-driver = { module = "com.datastax.cassandra:cassandra-driver-core",
version.ref = "cassandra-driver" }
failsafe = { module = "dev.failsafe:failsafe", version.ref = "failsafe" }
-docker-java-core = { module = "com.github.docker-java:docker-java-core",
version = "3.4.1" }
mongodb-driver-reactivestreams = { module =
"org.mongodb:mongodb-driver-reactivestreams", version.ref = "mongodb-driver" }
mongodb-driver-sync = { module = "org.mongodb:mongodb-driver-sync",
version.ref = "mongodb-driver" }
lettuce-core = { module = "io.lettuce:lettuce-core", version.ref = "lettuce" }
solr-solrj = { module = "org.apache.solr:solr-solrj", version.ref = "solr" }
solr-test-framework = { module = "org.apache.solr:solr-test-framework",
version.ref = "solr" }
solr-core = { module = "org.apache.solr:solr-core", version.ref = "solr" }
-
# Messaging
rabbitmq-amqp-client = { module = "com.rabbitmq:amqp-client", version.ref =
"rabbitmq-client" }
nsq-j = { module = "com.sproutsocial:nsq-j", version.ref = "nsq-client" }
-
# Time series
influxdb-client-java = { module = "com.influxdb:influxdb-client-java",
version.ref = "influxdb-client" }
influxdb-java = { module = "org.influxdb:influxdb-java", version.ref =
"influxdb-java" }
-
# IO specific
-jackson-dataformat-cbor = { module =
"com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", version.ref =
"jackson" }
-json-smart = { module = "net.minidev:json-smart", version = "2.5.2" }
-json-flattener = { module = "com.github.wnameless.json:json-flattener",
version = "0.16.4" }
-flatbuffers-java = { module = "com.google.flatbuffers:flatbuffers-java",
version = "1.9.0" }
-jfairy = { module = "io.codearte.jfairy:jfairy", version = "0.5.9" }
-embedded-redis = { module = "com.github.kstyrc:embedded-redis", version =
"0.6" }
-
+json-smart = "net.minidev:json-smart:2.5.2"
+json-flattener = "com.github.wnameless.json:json-flattener:0.16.4"
+flatbuffers-java = "com.google.flatbuffers:flatbuffers-java:1.9.0"
+jfairy = "io.codearte.jfairy:jfairy:0.5.9"
+embedded-redis = "com.github.kstyrc:embedded-redis:0.6"
# Auth
athenz-zts-java-client = { module = "com.yahoo.athenz:athenz-zts-java-client",
version.ref = "athenz" }
athenz-cert-refresher = { module = "com.yahoo.athenz:athenz-cert-refresher",
version.ref = "athenz" }
athenz-auth-core = { module = "com.yahoo.athenz:athenz-auth-core", version.ref
= "athenz" }
athenz-zpe-java-client = { module = "com.yahoo.athenz:athenz-zpe-java-client",
version.ref = "athenz" }
-
# Misc
bcprov-jdk18on = { module = "org.bouncycastle:bcprov-jdk18on", version.ref =
"bouncycastle-bcprov" }
commons-logging = { module = "commons-logging:commons-logging", version.ref =
"commons-logging" }
commons-beanutils = { module = "commons-beanutils:commons-beanutils",
version.ref = "commons-beanutils" }
commons-configuration2 = { module =
"org.apache.commons:commons-configuration2", version.ref =
"commons-configuration2" }
bookkeeper-stats-api = { module =
"org.apache.bookkeeper.stats:bookkeeper-stats-api", version.ref = "bookkeeper" }
-datasketches-memory = { module =
"org.apache.datasketches:datasketches-memory", version = "2.2.0" }
-datasketches-java = { module = "org.apache.datasketches:datasketches-java",
version = "6.1.1" }
-
+datasketches-memory = "org.apache.datasketches:datasketches-memory:2.2.0"
+datasketches-java = "org.apache.datasketches:datasketches-java:6.1.1"
# Pulsar core modules (published Maven artifacts, used by connectors)
-pulsar-io-core = { module = "org.apache.pulsar:pulsar-io-core", version.ref =
"pulsar" }
-pulsar-io-common = { module = "org.apache.pulsar:pulsar-io-common",
version.ref = "pulsar" }
-pulsar-common = { module = "org.apache.pulsar:pulsar-common", version.ref =
"pulsar" }
-pulsar-client-api = { module = "org.apache.pulsar:pulsar-client-api",
version.ref = "pulsar" }
-pulsar-client = { module = "org.apache.pulsar:pulsar-client-original",
version.ref = "pulsar" }
-pulsar-client-admin = { module =
"org.apache.pulsar:pulsar-client-admin-original", version.ref = "pulsar" }
-pulsar-config-validation = { module =
"org.apache.pulsar:pulsar-config-validation", version.ref = "pulsar" }
-pulsar-functions-api = { module = "org.apache.pulsar:pulsar-functions-api",
version.ref = "pulsar" }
-pulsar-functions-instance = { module =
"org.apache.pulsar:pulsar-functions-instance", version.ref = "pulsar" }
-pulsar-functions-utils = { module =
"org.apache.pulsar:pulsar-functions-utils", version.ref = "pulsar" }
-pulsar-broker = { module = "org.apache.pulsar:pulsar-broker", version.ref =
"pulsar" }
-pulsar-broker-test = { module = "org.apache.pulsar:pulsar-broker", version.ref
= "pulsar" }
-pulsar-testmocks = { module = "org.apache.pulsar:testmocks", version.ref =
"pulsar" }
-pulsar-buildtools = { module = "org.apache.pulsar:buildtools", version.ref =
"pulsar" }
+pulsar-bom = { module = "org.apache.pulsar:pulsar-bom", version.ref = "pulsar"
}
+pulsar-io-core = { module = "org.apache.pulsar:pulsar-io-core" }
+pulsar-io-common = { module = "org.apache.pulsar:pulsar-io-common" }
+pulsar-common = { module = "org.apache.pulsar:pulsar-common" }
+pulsar-client-api = { module = "org.apache.pulsar:pulsar-client-api" }
+pulsar-client = { module = "org.apache.pulsar:pulsar-client-original" }
+pulsar-client-admin = { module =
"org.apache.pulsar:pulsar-client-admin-original" }
+pulsar-config-validation = { module =
"org.apache.pulsar:pulsar-config-validation" }
+pulsar-functions-api = { module = "org.apache.pulsar:pulsar-functions-api" }
+pulsar-functions-instance = { module =
"org.apache.pulsar:pulsar-functions-instance" }
+pulsar-functions-utils = { module = "org.apache.pulsar:pulsar-functions-utils"
}
+pulsar-broker = { module = "org.apache.pulsar:pulsar-broker" }
+pulsar-testmocks = { module = "org.apache.pulsar:testmocks" }
+pulsar-buildtools = { module = "org.apache.pulsar:buildtools" }
[plugins]
lightproto = { id = "io.streamnative.lightproto", version.ref = "lightproto" }
-nar = { id = "io.github.merlimat.nar", version = "0.1.3" }
-protobuf = { id = "com.google.protobuf", version = "0.9.6" }
+nar = "io.github.merlimat.nar:0.1.3"
+protobuf = "com.google.protobuf:0.9.6"
shadow = { id = "com.gradleup.shadow", version.ref = "shadow" }
-rat = { id = "org.nosphere.apache.rat", version = "0.8.1" }
-license = { id = "com.github.hierynomus.license", version = "0.16.1" }
+rat = "org.nosphere.apache.rat:0.8.1"
+spotless = { id = "com.diffplug.spotless", version.ref = "spotless" }
+idea-ext = { id = "org.jetbrains.gradle.plugin.idea-ext", version.ref =
"idea-ext" }
+version-catalog-update = "nl.littlerobots.version-catalog-update:1.1.0"
+versions = "com.github.ben-manes.versions:0.53.0"
+crlf = "com.github.vlsi.crlf:3.0.1"
diff --git a/hbase/build.gradle.kts b/hbase/build.gradle.kts
index caa37ad..be1f1e5 100644
--- a/hbase/build.gradle.kts
+++ b/hbase/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/hdfs3/build.gradle.kts b/hdfs3/build.gradle.kts
index 1f20430..82e2254 100644
--- a/hdfs3/build.gradle.kts
+++ b/hdfs3/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/http/build.gradle.kts b/http/build.gradle.kts
index f4008b9..9b101af 100644
--- a/http/build.gradle.kts
+++ b/http/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/influxdb/build.gradle.kts b/influxdb/build.gradle.kts
index a29d19b..d3a589e 100644
--- a/influxdb/build.gradle.kts
+++ b/influxdb/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.common)
diff --git a/jdbc/clickhouse/build.gradle.kts b/jdbc/clickhouse/build.gradle.kts
index da66bc4..7a36945 100644
--- a/jdbc/clickhouse/build.gradle.kts
+++ b/jdbc/clickhouse/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(project(":jdbc:pulsar-io-jdbc-core"))
diff --git a/jdbc/core/build.gradle.kts b/jdbc/core/build.gradle.kts
index 7d38cdf..5d40fba 100644
--- a/jdbc/core/build.gradle.kts
+++ b/jdbc/core/build.gradle.kts
@@ -17,6 +17,10 @@
* under the License.
*/
+plugins {
+ id("pulsar-connectors.java-conventions")
+}
+
dependencies {
api(libs.pulsar.io.common)
api(libs.pulsar.io.core)
diff --git a/jdbc/mariadb/build.gradle.kts b/jdbc/mariadb/build.gradle.kts
index 1691bff..ce5fe96 100644
--- a/jdbc/mariadb/build.gradle.kts
+++ b/jdbc/mariadb/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(project(":jdbc:pulsar-io-jdbc-core"))
diff --git a/jdbc/openmldb/build.gradle.kts b/jdbc/openmldb/build.gradle.kts
index a9894a8..7eae514 100644
--- a/jdbc/openmldb/build.gradle.kts
+++ b/jdbc/openmldb/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(project(":jdbc:pulsar-io-jdbc-core"))
diff --git a/jdbc/postgres/build.gradle.kts b/jdbc/postgres/build.gradle.kts
index 752366f..3aa3da1 100644
--- a/jdbc/postgres/build.gradle.kts
+++ b/jdbc/postgres/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(project(":jdbc:pulsar-io-jdbc-core"))
diff --git a/jdbc/sqlite/build.gradle.kts b/jdbc/sqlite/build.gradle.kts
index 4e1495a..66c92cf 100644
--- a/jdbc/sqlite/build.gradle.kts
+++ b/jdbc/sqlite/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(project(":jdbc:pulsar-io-jdbc-core"))
diff --git a/kafka-connect-adaptor-nar/build.gradle.kts
b/kafka-connect-adaptor-nar/build.gradle.kts
index 4ee233d..05ebf63 100644
--- a/kafka-connect-adaptor-nar/build.gradle.kts
+++ b/kafka-connect-adaptor-nar/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
nar {
narId.set("pulsar-io-kafka-connect-adaptor")
diff --git a/kafka-connect-adaptor/build.gradle.kts
b/kafka-connect-adaptor/build.gradle.kts
index 4809b82..1349adb 100644
--- a/kafka-connect-adaptor/build.gradle.kts
+++ b/kafka-connect-adaptor/build.gradle.kts
@@ -17,6 +17,9 @@
* under the License.
*/
+plugins {
+ id("pulsar-connectors.java-conventions")
+}
dependencies {
compileOnly(libs.pulsar.io.core)
diff --git a/kafka/build.gradle.kts b/kafka/build.gradle.kts
index 8046aa6..6d219d0 100644
--- a/kafka/build.gradle.kts
+++ b/kafka/build.gradle.kts
@@ -18,16 +18,14 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
-
// KafkaBytesSource uses SchemaInfoImpl from pulsar-common, which is excluded
from
-// NAR runtimeClasspath by the global exclusion. Bundle it via a separate
configuration
-// since the NAR classloader's parent (rootClassLoader) only has
java-instance.jar.
-val narExtraDeps by configurations.creating {
- isCanBeResolved = true
- isCanBeConsumed = false
+// NAR runtimeClasspath by default. Include it in the NAR bundle.
+pulsarConnectorsNar {
+ includePulsarModule("pulsar-common")
}
dependencies {
@@ -35,7 +33,6 @@ dependencies {
implementation(libs.pulsar.io.core)
implementation(libs.pulsar.common)
implementation(libs.pulsar.client)
- narExtraDeps(libs.pulsar.common)
implementation(libs.jackson.databind)
implementation(libs.jackson.dataformat.yaml)
implementation(libs.guava)
@@ -52,9 +49,3 @@ dependencies {
testImplementation(libs.awaitility)
testImplementation(libs.bcpkix.jdk18on)
}
-
-tasks.named<io.github.merlimat.gradle.nar.NarTask>("nar") {
- from(narExtraDeps) { into("META-INF/bundled-dependencies") }
- bundledDependencies.from(narExtraDeps)
- duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-}
diff --git a/kinesis-kpl-shaded/build.gradle.kts
b/kinesis-kpl-shaded/build.gradle.kts
index 4197e5d..bf6ecce 100644
--- a/kinesis-kpl-shaded/build.gradle.kts
+++ b/kinesis-kpl-shaded/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.shadow)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.shadow-conventions")
}
dependencies {
@@ -34,16 +35,7 @@ configurations.all {
}
}
-// Disable the default jar task so the shadow JAR is the only artifact.
-// This avoids Gradle's implicit dependency validation errors when consumers
-// use project() to depend on this module.
-tasks.jar {
- enabled = false
-}
-
tasks.shadowJar {
- archiveClassifier.set("")
- mergeServiceFiles()
dependencies {
include(dependency("software.amazon.kinesis:amazon-kinesis-producer"))
include(dependency("com.google.protobuf:protobuf-java"))
diff --git a/kinesis/build.gradle.kts b/kinesis/build.gradle.kts
index 53f4995..2a80133 100644
--- a/kinesis/build.gradle.kts
+++ b/kinesis/build.gradle.kts
@@ -18,13 +18,14 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
// The shaded KPL project bundles amazon-kinesis-producer with relocated
protobuf.
- // Use shadowElements to get the shadow JAR (which has relocated protobuf).
- implementation(project(path = ":kinesis-kpl-shaded", configuration =
"shadowElements"))
+ // The shadow convention exposes the shadow JAR as the primary artifact.
+ implementation(project(":kinesis-kpl-shaded"))
// CompileOnly: needed for compilation against KPL classes but NOT bundled
in NAR.
// At runtime, KPL classes come from the shaded project's shadow JAR.
compileOnly(libs.amazon.kinesis.producer)
diff --git a/mongo/build.gradle.kts b/mongo/build.gradle.kts
index b760c25..d06d6c8 100644
--- a/mongo/build.gradle.kts
+++ b/mongo/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.common)
diff --git a/netty/build.gradle.kts b/netty/build.gradle.kts
index be70778..43e3e5a 100644
--- a/netty/build.gradle.kts
+++ b/netty/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/nsq/build.gradle.kts b/nsq/build.gradle.kts
index 732c22c..be75a00 100644
--- a/nsq/build.gradle.kts
+++ b/nsq/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.core)
diff --git a/pulsar-dependencies/build.gradle.kts
b/pulsar-connectors-dependencies/build.gradle.kts
similarity index 55%
rename from pulsar-dependencies/build.gradle.kts
rename to pulsar-connectors-dependencies/build.gradle.kts
index 7bead99..0be83b0 100644
--- a/pulsar-dependencies/build.gradle.kts
+++ b/pulsar-connectors-dependencies/build.gradle.kts
@@ -19,7 +19,7 @@
// Enforced platform module that declares version constraints for all
dependencies.
// This is the Gradle equivalent of Maven's dependencyManagement section.
-// All subprojects consume this via:
implementation(enforcedPlatform(project(":pulsar-dependencies")))
+// All subprojects consume this via:
implementation(enforcedPlatform(project(":pulsar-connectors-dependencies")))
plugins {
`java-platform`
}
@@ -30,14 +30,20 @@ javaPlatform {
}
dependencies {
- constraints {
- // Iterate over all library declarations in the version catalog and
add them as constraints.
- // This ensures that any transitive dependency matching a catalog
entry gets pinned to
- // the version we specify, regardless of what version a transitive
dependency requests.
- val catalog =
project.extensions.getByType<VersionCatalogsExtension>().named("libs")
- catalog.libraryAliases.forEach { alias ->
- catalog.findLibrary(alias).ifPresent { provider ->
- api(provider)
+ val catalog =
project.extensions.getByType<VersionCatalogsExtension>().named("libs")
+ // Iterate over all library declarations in the version catalog and add
them as constraints.
+ // This ensures that any transitive dependency matching a catalog entry
gets pinned to
+ // the version we specify, regardless of what version a transitive
dependency requests.
+ // BOM entries (detected by alias name) are imported as platforms rather
than constraints.
+ catalog.libraryAliases.forEach { alias ->
+ catalog.findLibrary(alias).ifPresent { provider ->
+ val dep = provider.get()
+ if (alias.endsWith(".bom")) {
+ api(platform(provider))
+ } else if (dep.versionConstraint.requiredVersion.isNotEmpty()) {
+ // Only add constraints for entries with explicit versions.
+ // Versionless entries are managed by BOMs imported above.
+ constraints.api(provider)
}
}
}
diff --git a/rabbitmq/build.gradle.kts b/rabbitmq/build.gradle.kts
index d7613ce..b15ee74 100644
--- a/rabbitmq/build.gradle.kts
+++ b/rabbitmq/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.common)
diff --git a/redis/build.gradle.kts b/redis/build.gradle.kts
index eb57bee..bfd82a9 100644
--- a/redis/build.gradle.kts
+++ b/redis/build.gradle.kts
@@ -18,7 +18,8 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
dependencies {
implementation(libs.pulsar.io.common)
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 9194ce0..d186f01 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -22,6 +22,7 @@ pluginManagement {
gradlePluginPortal()
mavenCentral()
}
+ includeBuild("build-logic")
}
dependencyResolutionManagement {
@@ -49,8 +50,16 @@ dependencyResolutionManagement {
rootProject.name = "pulsar-connectors"
+// This build requires Java 17 or later. Version check can be skipped with
-PskipJavaVersionCheck parameter.
+val javaVersion = providers.provider { JavaVersion.current() }
+require(providers.gradleProperty("skipJavaVersionCheck").isPresent
+ || javaVersion.get() >= JavaVersion.VERSION_17) {
+ "This build requires Java 17 or later, but is running on Java
${javaVersion.get()}. " +
+ "Pass -PskipJavaVersionCheck to skip this check."
+}
+
// Enforced platform for dependency version management
-include("pulsar-dependencies")
+include("pulsar-connectors-dependencies")
// Simple connectors (flat layout, top-level directories)
include("aerospike")
diff --git a/solr/build.gradle.kts b/solr/build.gradle.kts
index d15cedc..d338d67 100644
--- a/solr/build.gradle.kts
+++ b/solr/build.gradle.kts
@@ -18,10 +18,11 @@
*/
plugins {
- alias(libs.plugins.nar)
+ id("pulsar-connectors.java-conventions")
+ id("pulsar-connectors.nar-conventions")
}
// Solr 9.x embeds Jetty 10.x, which is incompatible with Pulsar's Jetty 12.
-// The pulsar-dependencies platform enforces Jetty 12 strict versions, which
override
+// The pulsar-connectors-dependencies platform enforces Jetty 12 strict
versions, which override
// enforcedPlatform("jetty-bom:10.0.24") because Gradle picks the highest
strict version.
// Use resolutionStrategy.force to downgrade Jetty to 10.0.24 for test
configurations.
val jetty10Version = "10.0.24"
diff --git a/src/license-header.txt b/src/license-header.txt
new file mode 100644
index 0000000..60b675e
--- /dev/null
+++ b/src/license-header.txt
@@ -0,0 +1,16 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
