This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 6e7ba7cb09 [python] Add DataFusion extra for merge conditions (#9463)
6e7ba7cb09 is described below
commit 6e7ba7cb09f0d9ce0bfb9e820a07061f6df22381
Author: XiaoHongbo <[email protected]>
AuthorDate: Sat Aug 29 20:53:57 2026 +0800
[python] Add DataFusion extra for merge conditions (#9463)
---
docs/docs/pypaimon/data-evolution.md | 5 +++--
docs/docs/pypaimon/multimodal-api.mdx | 5 +++--
docs/docs/pypaimon/ray-data.md | 5 +++--
paimon-python/pypaimon/ray/merge_condition.py | 10 +++++++---
.../tests/ray_data_evolution_merge_into_test.py | 2 +-
paimon-python/pypaimon/tests/table_merge_into_test.py | 17 ++++++++++++++++-
paimon-python/setup.py | 3 +++
7 files changed, 36 insertions(+), 11 deletions(-)
diff --git a/docs/docs/pypaimon/data-evolution.md
b/docs/docs/pypaimon/data-evolution.md
index c8cce6df9f..0a8a52515f 100644
--- a/docs/docs/pypaimon/data-evolution.md
+++ b/docs/docs/pypaimon/data-evolution.md
@@ -448,8 +448,9 @@ messages = table_update.merge_into(
Conditions use SQL-style expressions with `s.` (source) and `t.` (target)
column prefixes. `WhenNotMatched` conditions may only reference source columns
-(`s.*`). Condition evaluation uses DataFusion through the PyPaimon SQL extra.
-Install the extra before using conditions: `pip install pypaimon[sql]`.
+(`s.*`). Condition evaluation uses the PyPaimon DataFusion extra.
+Python 3.10 or newer is required. Install it with
+`pip install 'pypaimon[datafusion]'`.
```python
messages = table_update.merge_into(
diff --git a/docs/docs/pypaimon/multimodal-api.mdx
b/docs/docs/pypaimon/multimodal-api.mdx
index 6ca8d85634..1ba9d8dd84 100644
--- a/docs/docs/pypaimon/multimodal-api.mdx
+++ b/docs/docs/pypaimon/multimodal-api.mdx
@@ -426,8 +426,9 @@ docs.merge("id") \
```
Clause predicates use `where` for update and delete clauses. Refer to source
-rows with `source.<column>` and target rows with `target.<column>`. Install
-`pypaimon[sql]` when using merge clause predicates.
+rows with `source.<column>` and target rows with `target.<column>`. Merge
clause
+predicates require Python 3.10 or newer and
+`pip install 'pypaimon[datafusion]'`.
When source and target key names differ, pass a mapping from target column to
source column:
diff --git a/docs/docs/pypaimon/ray-data.md b/docs/docs/pypaimon/ray-data.md
index af6386e009..f314c65554 100644
--- a/docs/docs/pypaimon/ray-data.md
+++ b/docs/docs/pypaimon/ray-data.md
@@ -486,8 +486,9 @@ merge_into(
Conditions use SQL-style expressions with `s.` (source) and `t.` (target)
column prefixes. `WhenNotMatched` conditions may only reference source
-columns (`s.*`). Condition evaluation uses DataFusion through the PyPaimon SQL
-extra. Install the extra before using conditions: `pip install pypaimon[sql]`.
+columns (`s.*`). Condition evaluation uses the PyPaimon DataFusion extra.
+Python 3.10 or newer is required. Install it with
+`pip install 'pypaimon[datafusion]'`.
- `update` / `delete` / `insert`: `WhenMatched.update(...)` updates matched
rows, `WhenMatched.delete()` deletes matched rows, and
diff --git a/paimon-python/pypaimon/ray/merge_condition.py
b/paimon-python/pypaimon/ray/merge_condition.py
index cfbb2ada80..8bd437707a 100644
--- a/paimon-python/pypaimon/ray/merge_condition.py
+++ b/paimon-python/pypaimon/ray/merge_condition.py
@@ -18,6 +18,7 @@
import logging
import re
+import sys
from typing import Mapping, Optional, Set
import pyarrow as pa
@@ -32,14 +33,17 @@ logger = logging.getLogger(__name__)
def _load_datafusion():
+ if sys.version_info[:2] < (3, 10):
+ raise ImportError(
+ "merge_into condition expressions require Python 3.10 or newer"
+ )
try:
import datafusion
return datafusion
except ImportError:
raise ImportError(
- "merge_into condition expressions require the PyPaimon SQL "
- "extra, which provides DataFusion support. Install it with: "
- "pip install pypaimon[sql]"
+ "merge_into condition expressions require DataFusion. "
+ "Install it with: pip install 'pypaimon[datafusion]'"
)
diff --git a/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py
b/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py
index 8a72558bb7..be9ddf0c79 100644
--- a/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py
+++ b/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py
@@ -41,7 +41,7 @@ except ImportError:
_HAS_DATAFUSION = False
_SKIP_CONDITION = not _HAS_DATAFUSION
-_SKIP_REASON = "pypaimon[sql] is required for condition expressions"
+_SKIP_REASON = "pypaimon[datafusion] is required for condition expressions"
_TEST_NUM_PARTITIONS = 2
diff --git a/paimon-python/pypaimon/tests/table_merge_into_test.py
b/paimon-python/pypaimon/tests/table_merge_into_test.py
index f69ddfe8b5..811de74d90 100644
--- a/paimon-python/pypaimon/tests/table_merge_into_test.py
+++ b/paimon-python/pypaimon/tests/table_merge_into_test.py
@@ -42,11 +42,26 @@ except ImportError:
_HAS_DATAFUSION = False
_SKIP_CONDITION = not _HAS_DATAFUSION
-_SKIP_REASON = "pypaimon[sql] is required for condition expressions"
+_SKIP_REASON = "pypaimon[datafusion] is required for condition expressions"
class TableMergeIntoTest(BatchModeMixin, DataEvolutionTestBase,
unittest.TestCase):
+ def test_missing_datafusion_error_suggests_datafusion_extra(self):
+ from pypaimon.ray.merge_condition import _load_datafusion
+
+ with patch.dict("sys.modules", {"datafusion": None}):
+ with self.assertRaisesRegex(
+ ImportError, r"pip install 'pypaimon\[datafusion\]'"):
+ _load_datafusion()
+
+ def test_datafusion_conditions_require_python_310(self):
+ from pypaimon.ray.merge_condition import _load_datafusion
+
+ with patch("sys.version_info", (3, 9)):
+ with self.assertRaisesRegex(ImportError, "Python 3.10 or newer"):
+ _load_datafusion()
+
def _read_sorted(self, table):
return self._read_all(table).sort_by("id").to_pydict()
diff --git a/paimon-python/setup.py b/paimon-python/setup.py
index bf83fe4386..9111730fb7 100644
--- a/paimon-python/setup.py
+++ b/paimon-python/setup.py
@@ -280,6 +280,9 @@ setup(
'datasketches>=4,<5; python_version<"3.9"',
'datasketches>=5,<6; python_version>="3.9"',
],
+ 'datafusion': [
+ 'datafusion>=54,<55; python_version>="3.10"',
+ ],
'sql': [
'pypaimon-rust>=0.3.0; python_version>="3.10"',
'datafusion>=54,<55; python_version>="3.10"',
