Repository: spark Updated Branches: refs/heads/branch-0.9 563bfe1dd -> e03af416c
SPARK-1917: fix PySpark import of scipy.special functions https://issues.apache.org/jira/browse/SPARK-1917 Author: Uri Laserson <[email protected]> Closes #866 from laserson/SPARK-1917 and squashes the following commits: d947e8c [Uri Laserson] Added test for scipy.special importing 1798bbd [Uri Laserson] SPARK-1917: fix PySpark import of scipy.special Conflicts: python/pyspark/tests.py Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e03af416 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e03af416 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e03af416 Branch: refs/heads/branch-0.9 Commit: e03af416cf6947359135299e5550108ecb8001b2 Parents: 563bfe1 Author: Uri Laserson <[email protected]> Authored: Sat May 31 14:59:09 2014 -0700 Committer: Matei Zaharia <[email protected]> Committed: Sat May 31 15:08:00 2014 -0700 ---------------------------------------------------------------------- python/pyspark/cloudpickle.py | 2 +- python/pyspark/tests.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/e03af416/python/pyspark/cloudpickle.py ---------------------------------------------------------------------- diff --git a/python/pyspark/cloudpickle.py b/python/pyspark/cloudpickle.py index 6a7c23a..eb5dbb8 100644 --- a/python/pyspark/cloudpickle.py +++ b/python/pyspark/cloudpickle.py @@ -933,7 +933,7 @@ def _change_cell_value(cell, newval): Note: These can never be renamed due to client compatibility issues""" def _getobject(modname, attribute): - mod = __import__(modname) + mod = __import__(modname, fromlist=[attribute]) return mod.__dict__[attribute] def _generateImage(size, mode, str_rep): http://git-wip-us.apache.org/repos/asf/spark/blob/e03af416/python/pyspark/tests.py ---------------------------------------------------------------------- diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index 5271045..c741bfb 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -33,6 +33,14 @@ from pyspark.files import SparkFiles from pyspark.java_gateway import SPARK_HOME from pyspark.serializers import read_int +_have_scipy = False +try: + import scipy.sparse + _have_scipy = True +except: + # No SciPy, but that's okay, we'll skip those tests + pass + class PySparkTestCase(unittest.TestCase): @@ -234,5 +242,22 @@ class TestDaemon(unittest.TestCase): from signal import SIGTERM self.do_termination_test(lambda daemon: os.kill(daemon.pid, SIGTERM)) + [email protected](not _have_scipy, "SciPy not installed") +class SciPyTests(PySparkTestCase): + """General PySpark tests that depend on scipy """ + + def test_serialize(self): + from scipy.special import gammaln + x = range(1, 5) + expected = map(gammaln, x) + observed = self.sc.parallelize(x).map(gammaln).collect() + self.assertEqual(expected, observed) + + if __name__ == "__main__": + if not _have_scipy: + print "NOTE: Skipping SciPy tests as it does not seem to be installed" unittest.main() + if not _have_scipy: + print "NOTE: SciPy tests were skipped as it does not seem to be installed"
