Repository: spark Updated Branches: refs/heads/master b28137764 -> fa79d346e
[SPARK-14896][SQL] Deprecate HiveContext in python ## What changes were proposed in this pull request? See title. ## How was this patch tested? PySpark tests. Author: Andrew Or <and...@databricks.com> Closes #12917 from andrewor14/deprecate-hive-context-python. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fa79d346 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fa79d346 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fa79d346 Branch: refs/heads/master Commit: fa79d346e1a79ceda6ccd20e74eb850e769556ea Parents: b281377 Author: Andrew Or <and...@databricks.com> Authored: Wed May 4 17:39:30 2016 -0700 Committer: Andrew Or <and...@databricks.com> Committed: Wed May 4 17:39:30 2016 -0700 ---------------------------------------------------------------------- python/pyspark/sql/column.py | 2 -- python/pyspark/sql/context.py | 9 ++++++++- python/pyspark/sql/streaming.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/fa79d346/python/pyspark/sql/column.py ---------------------------------------------------------------------- diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py index 43e9bae..90fb76f 100644 --- a/python/pyspark/sql/column.py +++ b/python/pyspark/sql/column.py @@ -418,8 +418,6 @@ class Column(object): >>> window = Window.partitionBy("name").orderBy("age").rowsBetween(-1, 1) >>> from pyspark.sql.functions import rank, min >>> # df.select(rank().over(window), min('age').over(window)) - - .. note:: Window functions is only supported with HiveContext in 1.4 """ from pyspark.sql.window import WindowSpec if not isinstance(window, WindowSpec): http://git-wip-us.apache.org/repos/asf/spark/blob/fa79d346/python/pyspark/sql/context.py ---------------------------------------------------------------------- diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py index 2096236..78ab2e8 100644 --- a/python/pyspark/sql/context.py +++ b/python/pyspark/sql/context.py @@ -17,6 +17,7 @@ from __future__ import print_function import sys +import warnings if sys.version >= '3': basestring = unicode = str @@ -434,7 +435,6 @@ class SQLContext(object): return ContinuousQueryManager(self._ssql_ctx.streams()) -# TODO(andrew): deprecate this class HiveContext(SQLContext): """A variant of Spark SQL that integrates with data stored in Hive. @@ -444,8 +444,15 @@ class HiveContext(SQLContext): :param sparkContext: The SparkContext to wrap. :param jhiveContext: An optional JVM Scala HiveContext. If set, we do not instantiate a new :class:`HiveContext` in the JVM, instead we make all calls to this object. + + .. note:: Deprecated in 2.0.0. Use SparkSession.builder.enableHiveSupport().getOrCreate(). """ + warnings.warn( + "HiveContext is deprecated in Spark 2.0.0. Please use " + + "SparkSession.builder.enableHiveSupport().getOrCreate() instead.", + DeprecationWarning) + def __init__(self, sparkContext, jhiveContext=None): if jhiveContext is None: sparkSession = SparkSession.withHiveSupport(sparkContext) http://git-wip-us.apache.org/repos/asf/spark/blob/fa79d346/python/pyspark/sql/streaming.py ---------------------------------------------------------------------- diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py index bf03fdc..8238b8e 100644 --- a/python/pyspark/sql/streaming.py +++ b/python/pyspark/sql/streaming.py @@ -221,7 +221,7 @@ def _test(): globs['os'] = os globs['sc'] = sc globs['sqlContext'] = SQLContext(sc) - globs['hiveContext'] = HiveContext(sc) + globs['hiveContext'] = HiveContext._createForTesting(sc) globs['df'] = \ globs['sqlContext'].read.format('text').stream('python/test_support/sql/streaming') --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
