This is an automated email from the ASF dual-hosted git repository.
Samrat002 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-aws.git
The following commit(s) were added to refs/heads/main by this push:
new 2433fb39 [FLINK-40808] Add PyPI publishing support for
apache-flink-connector-aws
2433fb39 is described below
commit 2433fb3904b7034c39776d0e010e48dfce8c044a
Author: madilrob <[email protected]>
AuthorDate: Tue Oct 13 23:01:20 2026 -0700
[FLINK-40808] Add PyPI publishing support for apache-flink-connector-aws
---
.github/workflows/publish-python.yml | 122 +++++++++++++++++++++
docs/content/docs/connectors/datastream/kinesis.md | 46 ++++++++
flink-python/MANIFEST.in | 1 +
flink-python/setup.py | 9 +-
4 files changed, 174 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/publish-python.yml
b/.github/workflows/publish-python.yml
new file mode 100644
index 00000000..af6ed5e8
--- /dev/null
+++ b/.github/workflows/publish-python.yml
@@ -0,0 +1,122 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: Publish Python Package to PyPI
+
+on:
+ release:
+ types: [published]
+ workflow_dispatch:
+ inputs:
+ publish_target:
+ description: 'Where to publish the package'
+ required: true
+ default: 'testpypi'
+ type: choice
+ options:
+ - 'testpypi'
+ - 'pypi'
+
+jobs:
+ build:
+ name: Build distribution
+ runs-on: ubuntu-latest
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v4
+
+ - name: Set up Java 11
+ uses: actions/setup-java@v4
+ with:
+ java-version: '11'
+ distribution: 'temurin'
+ cache: maven
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Install build dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install build twine
+
+ - name: Build Java artifacts first
+ run: |
+ mvn clean install -DskipTests -B --no-transfer-progress \
+ -pl
flink-connector-aws/flink-sql-connector-aws-kinesis-streams,flink-connector-aws/flink-sql-connector-aws-kinesis-firehose
\
+ -am
+
+ - name: Build Python package
+ working-directory: flink-python
+ run: python -m build
+
+ - name: Check package with twine
+ working-directory: flink-python
+ run: twine check dist/*
+
+ - name: Upload distribution artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: python-package-distributions
+ path: flink-python/dist/
+
+ publish-to-pypi:
+ name: Publish to PyPI
+ if: github.event_name == 'release' || (github.event_name ==
'workflow_dispatch' && github.event.inputs.publish_target == 'pypi')
+ needs: build
+ runs-on: ubuntu-latest
+ environment:
+ name: pypi
+ url: https://pypi.org/p/apache-flink-connector-aws
+ permissions:
+ id-token: write # Required for OIDC trusted publishing
+
+ steps:
+ - name: Download distribution artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ - name: Publish to PyPI
+ uses: pypa/gh-action-pypi-publish@release/v1
+
+ publish-to-testpypi:
+ name: Publish to TestPyPI
+ if: github.event_name == 'workflow_dispatch' &&
github.event.inputs.publish_target == 'testpypi'
+ needs: build
+ runs-on: ubuntu-latest
+ environment:
+ name: testpypi
+ url: https://test.pypi.org/p/apache-flink-connector-aws
+ permissions:
+ id-token: write # Required for OIDC trusted publishing
+
+ steps:
+ - name: Download distribution artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ - name: Publish to TestPyPI
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ repository-url: https://test.pypi.org/legacy/
diff --git a/docs/content/docs/connectors/datastream/kinesis.md
b/docs/content/docs/connectors/datastream/kinesis.md
index 4de7d355..1fbe16a4 100644
--- a/docs/content/docs/connectors/datastream/kinesis.md
+++ b/docs/content/docs/connectors/datastream/kinesis.md
@@ -39,6 +39,22 @@ For use in PyFlink jobs, use the following dependency:
{{< py_connector_download_link "kinesis" >}}
+### Python Package Installation
+
+For Python applications, install the connector package from PyPI:
+
+```bash
+pip install apache-flink-connector-aws
+```
+
+This package provides Python wrappers for `KinesisStreamsSource`,
`KinesisStreamsSink`, and `KinesisFirehoseSink`.
+The package requires `apache-flink` to be installed, which will be installed
automatically as a dependency.
+
+{{< hint info >}}
+The Python package wraps the Java connector via Py4J. You still need to
include the Java JAR in your Flink cluster's classpath
+or submit it with your job using the `--jarfile` option.
+{{< /hint >}}
+
## Kinesis Streams Source
The `KinesisStreamsSource` is an exactly-once, parallel streaming data source
based on the [FLIP-27 source
interface](https://cwiki.apache.org/confluence/display/FLINK/FLIP-27%3A+Refactor+Source+Interface).
@@ -99,6 +115,36 @@ val kinesisEvents = env.fromSource(kdsSource,
WatermarkStrategy.forMonotonousTim
.uid("custom-uid")
```
{{< /tab >}}
+{{< tab "Python" >}}
+```python
+from pyflink.common import Configuration, WatermarkStrategy, Duration
+from pyflink.common.serialization import SimpleStringSchema
+from pyflink.datastream import StreamExecutionEnvironment
+from pyflink.datastream.connectors.kinesis import KinesisStreamsSource,
KinesisShardAssigner
+
+# Configure the KinesisStreamsSource
+source_config = Configuration()
+# source_config.set_string("source.init.position", "TRIM_HORIZON") #
Optional, default is LATEST
+
+# Create a new KinesisStreamsSource to read from specified Kinesis Stream.
+kds_source = KinesisStreamsSource.builder() \
+
.set_stream_arn("arn:aws:kinesis:us-east-1:123456789012:stream/test-stream") \
+ .set_source_config(source_config) \
+ .set_deserialization_schema(SimpleStringSchema()) \
+ .set_kinesis_shard_assigner(KinesisShardAssigner.uniform_shard_assigner())
\
+ .build()
+
+env = StreamExecutionEnvironment.get_execution_environment()
+
+# Specify watermarking strategy and the name of the Kinesis Source operator.
+# Specify UID of operator in line with Flink best practice.
+kinesis_stream = env.from_source(
+ source=kds_source,
+
watermark_strategy=WatermarkStrategy.for_monotonous_timestamps().with_idleness(Duration.of_seconds(1)),
+ source_name="Kinesis source"
+).uid("custom-uid")
+```
+{{< /tab >}}
{{< /tabs >}}
The above is a simple example of using the `KinesisStreamsSource`.
diff --git a/flink-python/MANIFEST.in b/flink-python/MANIFEST.in
index 720decf9..d913347e 100644
--- a/flink-python/MANIFEST.in
+++ b/flink-python/MANIFEST.in
@@ -17,4 +17,5 @@
################################################################################
graft pyflink
+prune pyflink/datastream/connectors/tests
global-exclude *.py[cod] __pycache__ .DS_Store
diff --git a/flink-python/setup.py b/flink-python/setup.py
index ce3eeae9..1101bb5d 100644
--- a/flink-python/setup.py
+++ b/flink-python/setup.py
@@ -21,7 +21,7 @@ import io
import os
import sys
-from setuptools import setup
+from setuptools import setup, find_namespace_packages
from shutil import rmtree
from xml.etree import ElementTree as ET
@@ -148,9 +148,10 @@ setup(
long_description=long_description,
long_description_content_type='text/plain',
zip_safe=False,
- py_modules=[
- "pyflink"
- ],
+ packages=find_namespace_packages(
+ include=['pyflink', 'pyflink.*'],
+ exclude=['pyflink.datastream.connectors.tests',
+ 'pyflink.datastream.connectors.tests.*']),
classifiers=[
'Development Status :: 5 - Production/Stable',
'License :: OSI Approved :: Apache Software License',
