This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c2a2f043cb [python] Add Python 3.7 to CI (#8695)
c2a2f043cb is described below
commit c2a2f043cb0f802a6aa5dcdd3b9e58831795b28a
Author: XiaoHongbo <[email protected]>
AuthorDate: Sat Jul 18 12:06:07 2026 +0800
[python] Add Python 3.7 to CI (#8695)
---
.github/workflows/paimon-python-checks.yml | 24 +++-
paimon-python/dev/lint-python.sh | 9 +-
paimon-python/dev/requirements.txt | 3 +-
paimon-python/dev/run_mixed_tests.sh | 149 +++++++++++++--------
paimon-python/pypaimon/__init__.py | 2 +-
.../pypaimon/filesystem/hdfs_native_file_io.py | 5 +-
paimon-python/pypaimon/filesystem/local_file_io.py | 7 +-
.../pypaimon/filesystem/pyarrow_file_io.py | 24 ++--
paimon-python/pypaimon/manifest/__init__.py | 2 +-
.../pypaimon/manifest/fastavro_py36_compat.py | 17 ++-
.../pypaimon/tests/e2e/java_py_read_write_test.py | 3 +-
paimon-python/pypaimon/tests/file_io_test.py | 19 +--
paimon-python/setup.py | 4 +-
13 files changed, 162 insertions(+), 106 deletions(-)
diff --git a/.github/workflows/paimon-python-checks.yml
b/.github/workflows/paimon-python-checks.yml
index 8e92a9c1fb..af7c7883fa 100755
--- a/.github/workflows/paimon-python-checks.yml
+++ b/.github/workflows/paimon-python-checks.yml
@@ -43,14 +43,14 @@ concurrency:
cancel-in-progress: true
jobs:
- # Lint + test on 3.6 and 3.10 only (not every Python version).
+ # Lint + test on selected versions only (3.6 / 3.7 low end, 3.10 / 3.11
current), not every Python version.
lint-python:
runs-on: ubuntu-latest
container: "python:${{ matrix.python-version }}-slim"
strategy:
fail-fast: false
matrix:
- python-version: [ '3.6.15', '3.10', '3.11' ]
+ python-version: [ '3.6.15', '3.7', '3.10', '3.11' ]
steps:
- name: Checkout code
@@ -86,13 +86,20 @@ jobs:
mvn -version
- name: Install Rust toolchain
+ if: matrix.python-version != '3.7'
run: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s --
-y --default-toolchain stable --profile minimal
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- - name: Install paimon-full-text native dependency
+ - name: Install paimon-full-text dependency
+ shell: bash
run: |
- bash .github/workflows/install-paimon-full-text.sh native
+ # 3.7 runs no native full-text tests; the Java artifact is enough.
+ if [[ "${{ matrix.python-version }}" == "3.7" ]]; then
+ bash .github/workflows/install-paimon-full-text.sh java
+ else
+ bash .github/workflows/install-paimon-full-text.sh native
+ fi
- name: Verify Python version
run: python --version
@@ -125,6 +132,13 @@ jobs:
python --version
python -m pip install --no-cache-dir pyroaring
readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4'
'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12'
'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest
py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null
python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION
}}' -i https://pypi.org/simple/
+ elif [[ "${{ matrix.python-version }}" == "3.7" ]]; then
+ # 3.7 installs the version-pinned set declared for 3.7 in
dev/requirements.txt.
+ # ray/lance/daft/torch have no 3.7 wheels; those tests are
importorskip-ed (as on 3.6.15).
+ python -m pip install --upgrade pip
+ python -m pip install -r paimon-python/dev/requirements.txt
+ python -m pip install flake8==4.0.1 'pytest~=7.0' py4j==0.10.9.9
parameterized==0.9.0
+ python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION
}}' -i https://pypi.org/simple/
else
python -m pip install --upgrade pip
pip install torch --index-url https://download.pytorch.org/whl/cpu
@@ -138,7 +152,7 @@ jobs:
df -h
- name: Install paimon-full-text Python dependency
- if: matrix.python-version != '3.6.15'
+ if: matrix.python-version != '3.6.15' && matrix.python-version != '3.7'
shell: bash
run: |
python -m pip install "${PAIMON_FULL_TEXT_DIR}/python"
diff --git a/paimon-python/dev/lint-python.sh b/paimon-python/dev/lint-python.sh
index ffee46cbef..e9a54c365c 100755
--- a/paimon-python/dev/lint-python.sh
+++ b/paimon-python/dev/lint-python.sh
@@ -195,13 +195,14 @@ function pytest_check() {
PYTHON_VERSION=$(python -c "import sys;
print(f'{sys.version_info.major}.{sys.version_info.minor}')")
echo "Detected Python version: $PYTHON_VERSION"
- # Determine test directory based on Python version
- if [ "$PYTHON_VERSION" = "3.6" ]; then
+ # 3.6/3.7 run a curated core subset (their dep ceiling rules out the
+ # vector/index/multimodal/blob suites); 3.10+ run the full suite.
+ if [ "$PYTHON_VERSION" = "3.6" ] || [ "$PYTHON_VERSION" = "3.7" ]; then
TEST_DIR="pypaimon/tests/py36 pypaimon/tests/file_io_test.py"
- echo "Running tests for Python 3.6: $TEST_DIR"
+ echo "Running core test subset for Python $PYTHON_VERSION: $TEST_DIR"
else
TEST_DIR="pypaimon/tests pypaimon/acceptance
--ignore=pypaimon/tests/py36 --ignore=pypaimon/tests/e2e
--ignore=pypaimon/tests/torch_read_test.py"
- echo "Running tests for Python $PYTHON_VERSION (excluding py36):
pypaimon/tests pypaimon/acceptance --ignore=pypaimon/tests/py36"
+ echo "Running tests for Python $PYTHON_VERSION (excluding py36):
$TEST_DIR"
fi
# the return value of a pipeline is the status of the last command to exit
diff --git a/paimon-python/dev/requirements.txt
b/paimon-python/dev/requirements.txt
index 3415427248..dc9b4b4e91 100644
--- a/paimon-python/dev/requirements.txt
+++ b/paimon-python/dev/requirements.txt
@@ -29,7 +29,8 @@ pandas>=1.5,<3; python_version >= "3.9"
polars>=0.9,<1; python_version<"3.8"
polars>=1,<2; python_version=="3.8"
polars>=1.32,<2; python_version>="3.9"
-pyarrow>=6,<7; python_version < "3.8"
+pyarrow>=6,<7; python_version < "3.7"
+pyarrow>=7,<13; python_version >= "3.7" and python_version < "3.8"
pyarrow>=16,!=19.0.0,<20; python_version >= "3.8"
pyroaring<=0.3.3; python_version < "3.7"
pyroaring<=0.4.5; python_version == "3.7"
diff --git a/paimon-python/dev/run_mixed_tests.sh
b/paimon-python/dev/run_mixed_tests.sh
index 7d5767357d..6423b59e04 100755
--- a/paimon-python/dev/run_mixed_tests.sh
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -118,10 +118,13 @@ run_batched_java_write_tests() {
result=1
fi
- local
lance_tests="org.apache.paimon.JavaPyLanceE2ETest#testJavaWriteReadPkTableLance"
- lance_tests="${lance_tests}+testDataEvolutionWriteLance"
- if ! run_maven_test_batch "paimon-lance Java write tests" "paimon-lance"
"$lance_tests" -q; then
- result=1
+ # lance has no Python wheel on <3.8; its readers are all skipped there.
+ if [[ "$PYTHON_MINOR" -ge 8 ]]; then
+ local
lance_tests="org.apache.paimon.JavaPyLanceE2ETest#testJavaWriteReadPkTableLance"
+ lance_tests="${lance_tests}+testDataEvolutionWriteLance"
+ if ! run_maven_test_batch "paimon-lance Java write tests"
"paimon-lance" "$lance_tests" -q; then
+ result=1
+ fi
fi
if [[ "$PYTHON_MINOR" -ge 11 ]]; then
@@ -142,10 +145,13 @@ run_batched_java_write_tests() {
fi
fi
- local
lumina_tests="org.apache.paimon.lumina.index.JavaPyLuminaE2ETest#testLuminaVectorIndexWrite"
- lumina_tests="${lumina_tests}+testLuminaVectorWithBTreeIndexWrite"
- if ! run_maven_test_batch "paimon-lumina Java write tests" "paimon-lumina"
"$lumina_tests" -q; then
- result=1
+ # Lumina vector reads need BitMap64 (>=3.8); skip the write below it too.
+ if [[ "$PYTHON_MINOR" -ge 8 ]]; then
+ local
lumina_tests="org.apache.paimon.lumina.index.JavaPyLuminaE2ETest#testLuminaVectorIndexWrite"
+ lumina_tests="${lumina_tests}+testLuminaVectorWithBTreeIndexWrite"
+ if ! run_maven_test_batch "paimon-lumina Java write tests"
"paimon-lumina" "$lumina_tests" -q; then
+ result=1
+ fi
fi
if [[ "$PYTHON_MINOR" -ge 9 ]]; then
@@ -187,15 +193,17 @@ run_java_write_test() {
echo ""
- # Run the Java test method for lance format
- echo "Running Maven test for
JavaPyLanceE2ETest.testJavaWriteReadPkTableLance (Lance)..."
- echo "Note: Maven may download dependencies on first run, this may take a
while..."
+ # Run the Java test method for lance format (readers skipped on <3.8).
local lance_result=0
- if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testJavaWriteReadPkTableLance -pl
paimon-lance -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java write lance test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java write lance test failed${NC}"
- lance_result=1
+ if [[ "$PYTHON_MINOR" -ge 8 ]]; then
+ echo "Running Maven test for
JavaPyLanceE2ETest.testJavaWriteReadPkTableLance (Lance)..."
+ echo "Note: Maven may download dependencies on first run, this may
take a while..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testJavaWriteReadPkTableLance -pl
paimon-lance -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java write lance test completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java write lance test failed${NC}"
+ lance_result=1
+ fi
fi
if [[ $parquet_result -eq 0 && $lance_result -eq 0 ]]; then
@@ -261,15 +269,19 @@ run_java_read_test() {
echo ""
- # Run Java test for lance format in paimon-lance
- echo "Running Maven test for JavaPyLanceE2ETest.testReadPkTableLance (Java
Read Lance)..."
- echo "Note: Maven may download dependencies on first run, this may take a
while..."
+ # Java read Lance reads a Python-written table; Python skips lance on <3.8.
local lance_result=0
- if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testReadPkTableLance -pl
paimon-lance -Drun.e2e.tests=true -Dpython.version="$PYTHON_VERSION"; then
- echo -e "${GREEN}✓ Java read lance test completed successfully${NC}"
+ if [[ "$PYTHON_MINOR" -ge 8 ]]; then
+ echo "Running Maven test for JavaPyLanceE2ETest.testReadPkTableLance
(Java Read Lance)..."
+ echo "Note: Maven may download dependencies on first run, this may
take a while..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testReadPkTableLance -pl
paimon-lance -Drun.e2e.tests=true -Dpython.version="$PYTHON_VERSION"; then
+ echo -e "${GREEN}✓ Java read lance test completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java read lance test failed${NC}"
+ lance_result=1
+ fi
else
- echo -e "${RED}✗ Java read lance test failed${NC}"
- lance_result=1
+ echo -e "${YELLOW}⏭ Skipping Java read Lance (lance needs Python >=
3.8, current: $PYTHON_VERSION)${NC}"
fi
if [[ $parquet_result -eq 0 && $lance_result -eq 0 ]]; then
@@ -800,13 +812,15 @@ run_data_evolution_test() {
core_result=1
fi
- # Java write data evolution table (lance)
- echo "Running Maven test for
JavaPyLanceE2ETest.testDataEvolutionWriteLance..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testDataEvolutionWriteLance -pl
paimon-lance -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java data evolution write (lance) completed
successfully${NC}"
- else
- echo -e "${RED}✗ Java data evolution write (lance) failed${NC}"
- lance_result=1
+ # Java write data evolution table (lance) -- only read back on >=3.8.
+ if [[ "$PYTHON_MINOR" -ge 8 ]]; then
+ echo "Running Maven test for
JavaPyLanceE2ETest.testDataEvolutionWriteLance..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testDataEvolutionWriteLance -pl
paimon-lance -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java data evolution write (lance) completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java data evolution write (lance) failed${NC}"
+ lance_result=1
+ fi
fi
fi
@@ -876,14 +890,18 @@ run_data_evolution_py_write_test() {
core_result=1
fi
- # Java read data evolution table (lance)
- echo "Running Maven test for
JavaPyLanceE2ETest.testReadDataEvolutionTableLance..."
+ # Java read Lance reads a Python-written table; Python skips lance on <3.8.
local lance_result=0
- if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testReadDataEvolutionTableLance -pl
paimon-lance -q -Drun.e2e.tests=true -Dpython.version="$PYTHON_VERSION"; then
- echo -e "${GREEN}✓ Java data evolution read (lance) completed
successfully${NC}"
+ if [[ "$PYTHON_MINOR" -ge 8 ]]; then
+ echo "Running Maven test for
JavaPyLanceE2ETest.testReadDataEvolutionTableLance..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testReadDataEvolutionTableLance -pl
paimon-lance -q -Drun.e2e.tests=true -Dpython.version="$PYTHON_VERSION"; then
+ echo -e "${GREEN}✓ Java data evolution read (lance) completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java data evolution read (lance) failed${NC}"
+ lance_result=1
+ fi
else
- echo -e "${RED}✗ Java data evolution read (lance) failed${NC}"
- lance_result=1
+ echo -e "${YELLOW}⏭ Skipping Java data evolution read Lance (lance
needs Python >= 3.8, current: $PYTHON_VERSION)${NC}"
fi
if [[ $core_result -ne 0 || $lance_result -ne 0 ]]; then
@@ -1151,22 +1169,31 @@ main() {
echo ""
- # Run BTree raw fallback test (Java write indexed + unindexed rows, Python
read)
- if ! run_btree_raw_fallback_test; then
- btree_raw_fallback_result=1
- fi
+ # Raw fallback / bitmap / global index need pyroaring BitMap64 (>=1.0,
+ # Python >=3.8) on the Python read side; skip on older interpreters.
+ if [[ "$PYTHON_MINOR" -ge 8 ]]; then
+ # Run BTree raw fallback test (Java write indexed + unindexed rows,
Python read)
+ if ! run_btree_raw_fallback_test; then
+ btree_raw_fallback_result=1
+ fi
- echo ""
+ echo ""
- # Run Bitmap index test (Java write, Python read)
- if ! run_bitmap_index_test; then
- bitmap_index_result=1
- fi
+ # Run Bitmap index test (Java write, Python read)
+ if ! run_bitmap_index_test; then
+ bitmap_index_result=1
+ fi
- echo ""
+ echo ""
- if ! run_compressed_global_index_test; then
- compressed_global_index_result=1
+ if ! run_compressed_global_index_test; then
+ compressed_global_index_result=1
+ fi
+ else
+ echo -e "${YELLOW}⏭ Skipping BTree raw fallback / bitmap / global
index tests (need BitMap64, Python >= 3.8, current: $PYTHON_VERSION)${NC}"
+ btree_raw_fallback_result=0
+ bitmap_index_result=0
+ compressed_global_index_result=0
fi
echo ""
@@ -1233,16 +1260,23 @@ main() {
echo ""
- # Run Lumina vector index test (Java write, Python read)
- if ! run_lumina_vector_test; then
- lumina_vector_result=1
- fi
+ # Lumina vector index needs BitMap64 (Python >= 3.8) on the read side.
+ if [[ "$PYTHON_MINOR" -ge 8 ]]; then
+ # Run Lumina vector index test (Java write, Python read)
+ if ! run_lumina_vector_test; then
+ lumina_vector_result=1
+ fi
- echo ""
+ echo ""
- # Run Lumina vector + BTree pre-filter test (Java write, Python read)
- if ! run_lumina_vector_btree_test; then
- lumina_vector_btree_result=1
+ # Run Lumina vector + BTree pre-filter test (Java write, Python read)
+ if ! run_lumina_vector_btree_test; then
+ lumina_vector_btree_result=1
+ fi
+ else
+ echo -e "${YELLOW}⏭ Skipping Lumina Vector Index tests (need BitMap64,
Python >= 3.8, current: $PYTHON_VERSION)${NC}"
+ lumina_vector_result=0
+ lumina_vector_btree_result=0
fi
echo ""
@@ -1293,7 +1327,8 @@ main() {
echo ""
- # Run data evolution test (Java write, Python read)
+ # Run data evolution test (Java write, Python read). Lance variant skips
+ # itself on <3.8 (get_file_format_params + gated Java lance read).
if ! run_data_evolution_test; then
data_evolution_result=1
fi
diff --git a/paimon-python/pypaimon/__init__.py
b/paimon-python/pypaimon/__init__.py
index 8f44e62537..aa18f812e7 100644
--- a/paimon-python/pypaimon/__init__.py
+++ b/paimon-python/pypaimon/__init__.py
@@ -17,7 +17,7 @@
import sys
-if sys.version_info[:2] == (3, 6):
+if sys.version_info[:2] < (3, 8):
try:
from pypaimon.manifest import fastavro_py36_compat # noqa: F401
except ImportError:
diff --git a/paimon-python/pypaimon/filesystem/hdfs_native_file_io.py
b/paimon-python/pypaimon/filesystem/hdfs_native_file_io.py
index 3c3d45ad86..78cad2140e 100644
--- a/paimon-python/pypaimon/filesystem/hdfs_native_file_io.py
+++ b/paimon-python/pypaimon/filesystem/hdfs_native_file_io.py
@@ -562,13 +562,14 @@ class HdfsNativeFileIO(FileIO):
def write_orc(self, path: str, data: pyarrow.Table,
compression: str = 'zstd', zstd_level: int = 1, **kwargs):
try:
- import sys
import pyarrow.orc as orc
data = self._cast_time_columns_for_orc(data)
with self.new_output_stream(path) as raw_stream:
stream = pyarrow.PythonFile(raw_stream, mode='wb')
try:
- if sys.version_info[:2] == (3, 6):
+ # ORC compression= was added in PyArrow 7.0; PyArrow 6
lacks it.
+ from pypaimon.filesystem.pyarrow_file_io import
_pyarrow_lt_7
+ if _pyarrow_lt_7():
orc.write_table(data, stream, **kwargs)
else:
orc.write_table(
diff --git a/paimon-python/pypaimon/filesystem/local_file_io.py
b/paimon-python/pypaimon/filesystem/local_file_io.py
index c9848705aa..c35e34d19d 100644
--- a/paimon-python/pypaimon/filesystem/local_file_io.py
+++ b/paimon-python/pypaimon/filesystem/local_file_io.py
@@ -307,9 +307,8 @@ class LocalFileIO(FileIO):
def write_orc(self, path: str, data: pyarrow.Table, compression: str =
'zstd',
zstd_level: int = 1, **kwargs):
try:
- import sys
import pyarrow.orc as orc
-
+
file_path = self._to_file(path)
parent = file_path.parent
if parent and not parent.exists():
@@ -318,7 +317,9 @@ class LocalFileIO(FileIO):
data = self._cast_time_columns_for_orc(data)
with open(file_path, 'wb') as f:
- if sys.version_info[:2] == (3, 6):
+ # ORC compression= was added in PyArrow 7.0; PyArrow 6 lacks
it.
+ from pypaimon.filesystem.pyarrow_file_io import _pyarrow_lt_7
+ if _pyarrow_lt_7():
orc.write_table(data, f, **kwargs)
else:
orc.write_table(data, f, compression=compression, **kwargs)
diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
index c9a3a6437c..8b0b4a7446 100644
--- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
+++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py
@@ -49,8 +49,12 @@ class PyArrowFileIO(FileIO):
def __init__(self, path: str, catalog_options: Options):
self.properties = catalog_options
self.logger = logging.getLogger(__name__)
- self._pyarrow_gte_7 = not _pyarrow_lt_7()
self._pyarrow_gte_8 = parse(pyarrow.__version__) >= parse("8.0.0")
+ # force_virtual_addressing landed in PyArrow 16; below it the OSS
bucket
+ # goes into endpoint_override, so keys must omit it (init + path share
+ # this flag so they can't drift).
+ self._pyarrow_gte_16 = parse(pyarrow.__version__) >= parse("16.0.0")
+ self._oss_bucket_in_endpoint = not self._pyarrow_gte_16
scheme, netloc, _ = self.parse_location(path)
self.uri_reader_factory = UriReaderFactory(catalog_options)
self._is_oss = scheme in {"oss"}
@@ -185,7 +189,7 @@ class PyArrowFileIO(FileIO):
"region": self.properties.get(OssOptions.OSS_REGION),
}
- if self._pyarrow_gte_7:
+ if not self._oss_bucket_in_endpoint:
client_kwargs['force_virtual_addressing'] = True
client_kwargs['endpoint_override'] =
self.properties.get(OssOptions.OSS_ENDPOINT)
else:
@@ -226,7 +230,7 @@ class PyArrowFileIO(FileIO):
"session_token": session_token,
"region": region,
}
- if self._pyarrow_gte_7:
+ if self._pyarrow_gte_16:
path_style_access = (
self._get_s3_boolean_property("path-style-access") or
self._get_s3_boolean_property("path.style.access"))
@@ -346,8 +350,8 @@ class PyArrowFileIO(FileIO):
if self._use_jindo:
pass
- elif self._is_oss and not self._pyarrow_gte_7:
- # For PyArrow 6.x + OSS, path_str is already just the key part
+ elif self._is_oss and self._oss_bucket_in_endpoint:
+ # OSS with bucket baked into endpoint: path_str is already the key
if '/' in path_str:
parent_dir = '/'.join(path_str.split('/')[:-1])
else:
@@ -554,15 +558,13 @@ class PyArrowFileIO(FileIO):
(which is 3, see
https://github.com/facebook/zstd/blob/dev/programs/zstdcli.c)
instead of the specified level.
"""
- import sys
-
import pyarrow.orc as orc
data = self._cast_time_columns_for_orc(data)
with self.new_output_stream(path) as output_stream:
- # Check Python version - if 3.6, don't use compression
parameter
- if sys.version_info[:2] == (3, 6):
+ # ORC compression= was added in PyArrow 7.0; PyArrow 6 lacks
it.
+ if _pyarrow_lt_7():
orc.write_table(data, output_stream, **kwargs)
else:
orc.write_table(
@@ -732,8 +734,8 @@ class PyArrowFileIO(FileIO):
if parsed.scheme:
if parsed.netloc:
path_part = normalized_path.lstrip('/')
- # OSS+PyArrow<7: endpoint_override has bucket, pass key
only.
- if self._is_oss and not self._pyarrow_gte_7:
+ # OSS with bucket baked into endpoint: pass key only.
+ if self._is_oss and self._oss_bucket_in_endpoint:
return path_part if path_part else '.'
result = f"{parsed.netloc}/{path_part}" if path_part else
parsed.netloc
return result
diff --git a/paimon-python/pypaimon/manifest/__init__.py
b/paimon-python/pypaimon/manifest/__init__.py
index 88064e3a67..8509db9163 100644
--- a/paimon-python/pypaimon/manifest/__init__.py
+++ b/paimon-python/pypaimon/manifest/__init__.py
@@ -18,7 +18,7 @@
# Apply fastavro Python 3.6 compatibility patch early, before any other
# manifest modules are imported that might use fastavro
import sys
-if sys.version_info[:2] == (3, 6):
+if sys.version_info[:2] < (3, 8):
try:
from pypaimon.manifest import fastavro_py36_compat # noqa: F401
except ImportError:
diff --git a/paimon-python/pypaimon/manifest/fastavro_py36_compat.py
b/paimon-python/pypaimon/manifest/fastavro_py36_compat.py
index 87b34b7d02..aba8c758a8 100644
--- a/paimon-python/pypaimon/manifest/fastavro_py36_compat.py
+++ b/paimon-python/pypaimon/manifest/fastavro_py36_compat.py
@@ -16,15 +16,14 @@
# under the License.
"""
-Provides compatibility patches for fastavro on Python 3.6,
-specifically for handling zstd-compressed Avro files.
+Compatibility patch for fastavro on Python 3.6/3.7 (the python-zstandard
+fastavro), for reading zstd-compressed Avro (manifest) files.
-The main issue addressed is:
-- On Python 3.6, fastavro's zstd decompression may fail with:
+Those fastavro versions may fail on frames without a content-size header
+(e.g. written by Java or backports.zstd) with:
"zstd.ZstdError: could not determine content size in frame header"
-
-This module patches fastavro's zstd handling to use a more compatible
-decompression method that works on Python 3.6.
+This patches fastavro's zstd block reader to stream-decompress instead.
+Python 3.8+ uses backports.zstd and needs no patch.
"""
import sys
@@ -34,7 +33,7 @@ _patch_applied = False
def _apply_zstd_patch():
global _patch_applied
- if _patch_applied or sys.version_info[:2] != (3, 6):
+ if _patch_applied or sys.version_info[:2] >= (3, 8):
return
try:
@@ -69,7 +68,7 @@ def _apply_zstd_patch():
_patch_applied = True
-if sys.version_info[:2] == (3, 6):
+if sys.version_info[:2] < (3, 8):
try:
_apply_zstd_patch()
except ImportError:
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index db7ace5243..ac9f44cc22 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -39,7 +39,8 @@ else:
def get_file_format_params():
- if sys.version_info[:2] == (3, 6):
+ # lance has no wheel on Python < 3.8.
+ if sys.version_info[:2] < (3, 8):
return [('parquet',), ('orc',), ('avro',)]
else:
return [('parquet',), ('orc',), ('avro',), ('lance',)]
diff --git a/paimon-python/pypaimon/tests/file_io_test.py
b/paimon-python/pypaimon/tests/file_io_test.py
index a9489cfd98..fd5f00306c 100644
--- a/paimon-python/pypaimon/tests/file_io_test.py
+++ b/paimon-python/pypaimon/tests/file_io_test.py
@@ -65,25 +65,26 @@ class FileIOTest(unittest.TestCase):
parent_str = str(Path(converted_path).parent)
self.assertEqual(file_io.to_filesystem_path(parent_str), parent_str)
- lt7 = _pyarrow_lt_7()
oss_io = PyArrowFileIO("oss://test-bucket/warehouse", Options({
OssOptions.OSS_ENDPOINT.key(): 'oss-cn-hangzhou.aliyuncs.com',
OssOptions.OSS_ACCESS_KEY_ID.key(): 'test-key',
OssOptions.OSS_ACCESS_KEY_SECRET.key(): 'test-secret',
OssOptions.OSS_IMPL.key(): 'legacy',
}))
+ # PyArrow <16 bakes the bucket into endpoint_override, so keys omit it.
+ bucket_stripped = oss_io._oss_bucket_in_endpoint
got = oss_io.to_filesystem_path("oss://test-bucket/path/to/file.txt")
- self.assertEqual(got, "path/to/file.txt" if lt7 else
"test-bucket/path/to/file.txt")
- if lt7:
+ self.assertEqual(got, "path/to/file.txt" if bucket_stripped else
"test-bucket/path/to/file.txt")
+ if bucket_stripped:
self.assertEqual(oss_io.to_filesystem_path("db-xxx.db/tbl-xxx/data.parquet"),
"db-xxx.db/tbl-xxx/data.parquet")
self.assertEqual(oss_io.to_filesystem_path("db-xxx.db/tbl-xxx"),
"db-xxx.db/tbl-xxx")
manifest_uri =
"oss://test-bucket/warehouse/db.db/table/manifest/manifest-list-abc-0"
manifest_key = oss_io.to_filesystem_path(manifest_uri)
self.assertEqual(manifest_key,
"warehouse/db.db/table/manifest/manifest-list-abc-0",
- "OSS+PyArrow6 must pass key only to PyArrow so
manifest is written to correct bucket")
+ "OSS+PyArrow<16 must pass key only so manifest
lands in the right bucket")
self.assertFalse(manifest_key.startswith("test-bucket/"),
- "path must not start with bucket name or PyArrow
6 writes to wrong bucket")
+ "path must not start with bucket or PyArrow <16
writes to wrong bucket")
nf = MagicMock(type=pafs.FileType.NotFound)
get_file_info_calls = []
@@ -92,24 +93,24 @@ class FileIOTest(unittest.TestCase):
return [MagicMock(type=pafs.FileType.NotFound) for _ in paths]
mock_fs = MagicMock()
- mock_fs.get_file_info.side_effect = record_get_file_info if lt7 else
[[nf], [nf]]
+ mock_fs.get_file_info.side_effect = record_get_file_info if
bucket_stripped else [[nf], [nf]]
mock_fs.create_dir = MagicMock()
mock_fs.open_output_stream.return_value = MagicMock()
oss_io.filesystem = mock_fs
oss_io.new_output_stream("oss://test-bucket/path/to/file.txt")
mock_fs.create_dir.assert_called_once()
path_str =
oss_io.to_filesystem_path("oss://test-bucket/path/to/file.txt")
- if lt7:
+ if bucket_stripped:
expected_parent = '/'.join(path_str.split('/')[:-1]) if '/' in
path_str else ''
else:
expected_parent = "/".join(path_str.split("/")[:-1]) if "/" in
path_str else str(Path(path_str).parent)
self.assertEqual(mock_fs.create_dir.call_args[0][0], expected_parent)
- if lt7:
+ if bucket_stripped:
for call_paths in get_file_info_calls:
for p in call_paths:
self.assertFalse(
p.startswith("test-bucket/"),
- "OSS+PyArrow<7 must pass key only to get_file_info,
not bucket/key. Got: %r" % (p,)
+ "OSS+PyArrow<16 must pass key only to get_file_info,
not bucket/key. Got: %r" % (p,)
)
def test_exists(self):
diff --git a/paimon-python/setup.py b/paimon-python/setup.py
index c4a543f8fb..f66717fb91 100644
--- a/paimon-python/setup.py
+++ b/paimon-python/setup.py
@@ -152,7 +152,7 @@ setup(
},
extras_require={
'ray': [
- 'ray>=2.10,<3; python_version>="3.7"',
+ 'ray>=2.10,<3; python_version>="3.8"',
],
'torch': [
'torch',
@@ -175,7 +175,7 @@ setup(
'vortex-data==0.70.0; python_version>="3.11"',
],
'mosaic': [
- 'paimon-mosaic>=0.1.0',
+ 'paimon-mosaic>=0.1.0; python_version>="3.9"',
],
'lumina': [
'lumina-data>=0.1.0'
