This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 26d44f3922b4 Revert "[SPARK-57065][PYTHON][TEST] Add with_sql_conf
decorator in SQLTestUtils"
26d44f3922b4 is described below
commit 26d44f3922b423738760610a51c4daf6f4f58f8d
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu May 28 16:05:34 2026 +0800
Revert "[SPARK-57065][PYTHON][TEST] Add with_sql_conf decorator in
SQLTestUtils"
### What changes were proposed in this pull request?
This reverts commit 0f13b94bb8f4f1cdff6ad64d3d27c5995561cab9 (#56106),
which added the `with_sql_conf` class decorator in `pyspark.testing.sqlutils`
and migrated three parity files to use it.
### Why are the changes needed?
On second look the decorator doesn't pull its weight: the parity sites it
replaced are already concise enough, and we don't have a follow-up wave of
callsites lined up that would amortize the new abstraction. Reverting now keeps
the test utilities surface minimal; if a clearer use case emerges later we can
reintroduce something targeted.
### Does this PR introduce _any_ user-facing change?
No - test infrastructure only.
### How was this patch tested?
Existing tests. The revert restores the pre-#56106 `setUpClass` /
`tearDownClass` overrides in the three migrated parity files verbatim.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (model: claude-opus-4-7)
Closes #56183 from zhengruifeng/revert-with-sql-conf-decorator-dev5.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../connect/arrow/test_parity_arrow_python_udf.py | 68 +++++++++++++++++++---
.../sql/tests/connect/test_parity_column.py | 27 ++++++++-
.../tests/connect/test_parity_udf_combinations.py | 7 ++-
.../pyspark/sql/tests/connect/test_parity_utils.py | 13 -----
python/pyspark/sql/tests/test_utils.py | 14 +----
python/pyspark/testing/sqlutils.py | 46 ---------------
6 files changed, 88 insertions(+), 87 deletions(-)
diff --git
a/python/pyspark/sql/tests/connect/arrow/test_parity_arrow_python_udf.py
b/python/pyspark/sql/tests/connect/arrow/test_parity_arrow_python_udf.py
index eef455ff357f..0a7a54521686 100644
--- a/python/pyspark/sql/tests/connect/arrow/test_parity_arrow_python_udf.py
+++ b/python/pyspark/sql/tests/connect/arrow/test_parity_arrow_python_udf.py
@@ -19,16 +19,35 @@ import unittest
from pyspark.sql.tests.connect.test_parity_udf import UDFParityTests
from pyspark.sql.tests.arrow.test_arrow_python_udf import
ArrowPythonUDFTestsMixin
-from pyspark.testing.sqlutils import with_sql_conf
-@with_sql_conf({"spark.sql.execution.pythonUDF.arrow.enabled": "true"})
class ArrowPythonUDFParityTests(UDFParityTests, ArrowPythonUDFTestsMixin):
- pass
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ cls.spark.conf.set("spark.sql.execution.pythonUDF.arrow.enabled",
"true")
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+ cls.spark.conf.unset("spark.sql.execution.pythonUDF.arrow.enabled")
+ finally:
+ super().tearDownClass()
-@with_sql_conf({"spark.sql.legacy.execution.pythonUDF.pandas.conversion.enabled":
"true"})
class ArrowPythonUDFParityLegacyTestsMixin(ArrowPythonUDFTestsMixin):
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+
cls.spark.conf.set("spark.sql.legacy.execution.pythonUDF.pandas.conversion.enabled",
"true")
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+
cls.spark.conf.unset("spark.sql.legacy.execution.pythonUDF.pandas.conversion.enabled")
+ finally:
+ super().tearDownClass()
+
@unittest.skip("Duplicate test as it is already tested in
ArrowPythonUDFLegacyTests.")
def test_udf_binary_type(self):
super().test_udf_binary_type(self)
@@ -38,8 +57,21 @@ class
ArrowPythonUDFParityLegacyTestsMixin(ArrowPythonUDFTestsMixin):
super().test_udf_binary_type_in_nested_structures(self)
-@with_sql_conf({"spark.sql.legacy.execution.pythonUDF.pandas.conversion.enabled":
"false"})
class ArrowPythonUDFParityNonLegacyTestsMixin(ArrowPythonUDFTestsMixin):
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ cls.spark.conf.set(
+ "spark.sql.legacy.execution.pythonUDF.pandas.conversion.enabled",
"false"
+ )
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+
cls.spark.conf.unset("spark.sql.legacy.execution.pythonUDF.pandas.conversion.enabled")
+ finally:
+ super().tearDownClass()
+
@unittest.skip("Duplicate test as it is already tested in
ArrowPythonUDFNonLegacyTests.")
def test_udf_binary_type(self):
super().test_udf_binary_type(self)
@@ -49,14 +81,32 @@ class
ArrowPythonUDFParityNonLegacyTestsMixin(ArrowPythonUDFTestsMixin):
super().test_udf_binary_type_in_nested_structures(self)
-@with_sql_conf({"spark.sql.execution.pythonUDF.arrow.enabled": "true"})
class ArrowPythonUDFParityLegacyTests(UDFParityTests,
ArrowPythonUDFParityLegacyTestsMixin):
- pass
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ cls.spark.conf.set("spark.sql.execution.pythonUDF.arrow.enabled",
"true")
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+ cls.spark.conf.unset("spark.sql.execution.pythonUDF.arrow.enabled")
+ finally:
+ super().tearDownClass()
-@with_sql_conf({"spark.sql.execution.pythonUDF.arrow.enabled": "true"})
class ArrowPythonUDFParityNonLegacyTests(UDFParityTests,
ArrowPythonUDFParityNonLegacyTestsMixin):
- pass
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+ cls.spark.conf.set("spark.sql.execution.pythonUDF.arrow.enabled",
"true")
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+ cls.spark.conf.unset("spark.sql.execution.pythonUDF.arrow.enabled")
+ finally:
+ super().tearDownClass()
if __name__ == "__main__":
diff --git a/python/pyspark/sql/tests/connect/test_parity_column.py
b/python/pyspark/sql/tests/connect/test_parity_column.py
index a3f922127c10..3903bb57a375 100644
--- a/python/pyspark/sql/tests/connect/test_parity_column.py
+++ b/python/pyspark/sql/tests/connect/test_parity_column.py
@@ -19,11 +19,21 @@ import unittest
from pyspark.sql.tests.test_column import ColumnTestsMixin
from pyspark.testing.connectutils import ReusedConnectTestCase
-from pyspark.testing.sqlutils import with_sql_conf
-@with_sql_conf({"spark.sql.analyzer.strictDataFrameColumnResolution": "true"})
class ColumnParityTests(ColumnTestsMixin, ReusedConnectTestCase):
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+
cls.spark.conf.set("spark.sql.analyzer.strictDataFrameColumnResolution", "true")
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+
cls.spark.conf.unset("spark.sql.analyzer.strictDataFrameColumnResolution")
+ finally:
+ super().tearDownClass()
+
@unittest.skip("Requires JVM access.")
def test_validate_column_types(self):
super().test_validate_column_types()
@@ -35,12 +45,23 @@ class ColumnParityTests(ColumnTestsMixin,
ReusedConnectTestCase):
)
-@with_sql_conf({"spark.sql.analyzer.strictDataFrameColumnResolution": "false"})
class ColumnParityTestsWithNonStrictDFColResolution(ColumnParityTests):
"""Re-run the Column parity tests with
`spark.sql.analyzer.strictDataFrameColumnResolution=false` to exercise the
name-based fallback path for tagged UnresolvedAttributes."""
+ @classmethod
+ def setUpClass(cls):
+ super().setUpClass()
+
cls.spark.conf.set("spark.sql.analyzer.strictDataFrameColumnResolution",
"false")
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+
cls.spark.conf.unset("spark.sql.analyzer.strictDataFrameColumnResolution")
+ finally:
+ super().tearDownClass()
+
def test_df_col_resolution_mode(self):
self.assertEqual(
self.spark.conf.get("spark.sql.analyzer.strictDataFrameColumnResolution"),
diff --git a/python/pyspark/sql/tests/connect/test_parity_udf_combinations.py
b/python/pyspark/sql/tests/connect/test_parity_udf_combinations.py
index ae8ebb6368f9..b1e8d473201d 100644
--- a/python/pyspark/sql/tests/connect/test_parity_udf_combinations.py
+++ b/python/pyspark/sql/tests/connect/test_parity_udf_combinations.py
@@ -18,12 +18,13 @@
from pyspark.sql.tests.test_udf_combinations import UDFCombinationsTestsMixin
from pyspark.testing.connectutils import ReusedConnectTestCase
-from pyspark.testing.sqlutils import with_sql_conf
-@with_sql_conf({"spark.sql.execution.pythonUDF.arrow.enabled": "false"})
class UDFCombinationsParityTests(UDFCombinationsTestsMixin,
ReusedConnectTestCase):
- pass
+ @classmethod
+ def setUpClass(cls):
+ ReusedConnectTestCase.setUpClass()
+ cls.spark.conf.set("spark.sql.execution.pythonUDF.arrow.enabled",
"false")
if __name__ == "__main__":
diff --git a/python/pyspark/sql/tests/connect/test_parity_utils.py
b/python/pyspark/sql/tests/connect/test_parity_utils.py
index ed03243b8119..521b6082cf22 100644
--- a/python/pyspark/sql/tests/connect/test_parity_utils.py
+++ b/python/pyspark/sql/tests/connect/test_parity_utils.py
@@ -16,7 +16,6 @@
#
from pyspark.testing.connectutils import ReusedConnectTestCase
-from pyspark.testing.sqlutils import with_sql_conf
from pyspark.sql.tests.test_utils import UtilsTestsMixin
@@ -24,18 +23,6 @@ class UtilsParityTests(UtilsTestsMixin,
ReusedConnectTestCase):
pass
-@with_sql_conf(
- {
- "spark.sql.test.with_sql_conf.key1": "v1",
- "spark.sql.test.with_sql_conf.key2": "v2",
- }
-)
-class WithSqlConfParityTests(ReusedConnectTestCase):
- def test_confs_applied(self):
-
self.assertEqual(self.spark.conf.get("spark.sql.test.with_sql_conf.key1"), "v1")
-
self.assertEqual(self.spark.conf.get("spark.sql.test.with_sql_conf.key2"), "v2")
-
-
if __name__ == "__main__":
from pyspark.testing import main
diff --git a/python/pyspark/sql/tests/test_utils.py
b/python/pyspark/sql/tests/test_utils.py
index 6e5a0ee69287..3454a5f8b66c 100644
--- a/python/pyspark/sql/tests/test_utils.py
+++ b/python/pyspark/sql/tests/test_utils.py
@@ -37,7 +37,7 @@ from pyspark.testing.utils import (
have_pandas,
have_pyarrow,
)
-from pyspark.testing.sqlutils import ReusedSQLTestCase, with_sql_conf
+from pyspark.testing.sqlutils import ReusedSQLTestCase
from pyspark.sql import Row
import pyspark.sql.functions as F
from pyspark.sql.functions import to_date, unix_timestamp, from_unixtime
@@ -1876,18 +1876,6 @@ class UtilsTests(UtilsTestsMixin, ReusedSQLTestCase):
pass
-@with_sql_conf(
- {
- "spark.sql.test.with_sql_conf.key1": "v1",
- "spark.sql.test.with_sql_conf.key2": "v2",
- }
-)
-class WithSqlConfTests(ReusedSQLTestCase):
- def test_confs_applied(self):
-
self.assertEqual(self.spark.conf.get("spark.sql.test.with_sql_conf.key1"), "v1")
-
self.assertEqual(self.spark.conf.get("spark.sql.test.with_sql_conf.key2"), "v2")
-
-
if __name__ == "__main__":
from pyspark.testing import main
diff --git a/python/pyspark/testing/sqlutils.py
b/python/pyspark/testing/sqlutils.py
index f91cb9f85e0e..e9d99dd3d2d6 100644
--- a/python/pyspark/testing/sqlutils.py
+++ b/python/pyspark/testing/sqlutils.py
@@ -170,52 +170,6 @@ except Exception as e:
test_compiled = not test_not_compiled_message
-def with_sql_conf(pairs):
- """
- Class decorator that sets the given Spark confs in ``setUpClass`` and
unsets them in
- ``tearDownClass``, around the calls to the inherited
``setUpClass``/``tearDownClass``.
-
- The decorated class is expected to expose ``cls.spark`` (a
``SparkSession``) after the
- inherited ``setUpClass`` runs — typically by extending
``ReusedSQLTestCase`` or
- ``ReusedConnectTestCase``.
-
- If the decorated class defines its own ``setUpClass``/``tearDownClass``,
those are
- called instead of the inherited ones; the conf set/unset happens after
setUpClass
- and before tearDownClass respectively.
- """
- assert isinstance(pairs, dict), "pairs should be a dictionary."
-
- def decorator(cls):
- own_setup = cls.__dict__.get("setUpClass")
- own_teardown = cls.__dict__.get("tearDownClass")
-
- @classmethod
- def setUpClass(klass):
- if own_setup is not None:
- own_setup.__func__(klass)
- else:
- super(cls, klass).setUpClass()
- for key, value in pairs.items():
- klass.spark.conf.set(key, value)
-
- @classmethod
- def tearDownClass(klass):
- try:
- for key in pairs:
- klass.spark.conf.unset(key)
- finally:
- if own_teardown is not None:
- own_teardown.__func__(klass)
- else:
- super(cls, klass).tearDownClass()
-
- cls.setUpClass = setUpClass
- cls.tearDownClass = tearDownClass
- return cls
-
- return decorator
-
-
class SQLTestUtils:
"""
This util assumes the instance of this to have 'spark' attribute, having a
spark session.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
