This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 18925c3 [SPARK-31739][PYSPARK][DOCS][MINOR] Fix docstring syntax
issues and misplaced space characters
18925c3 is described below
commit 18925c34ffc403c1422dd8ac8ae731c213521387
Author: David Toneian <[email protected]>
AuthorDate: Mon May 18 20:25:02 2020 +0900
[SPARK-31739][PYSPARK][DOCS][MINOR] Fix docstring syntax issues and
misplaced space characters
This commit is published into the public domain.
Some syntax issues in docstrings have been fixed.
In some places, the documentation did not render as intended, e.g.
parameter documentations were not formatted as such.
Slight improvements in documentation.
Manual testing and `dev/lint-python` run. No new Sphinx warnings arise due
to this change.
Closes #28559 from DavidToneian/SPARK-31739.
Authored-by: David Toneian <[email protected]>
Signed-off-by: HyukjinKwon <[email protected]>
---
python/pyspark/ml/clustering.py | 2 +-
python/pyspark/ml/regression.py | 2 +-
python/pyspark/ml/util.py | 1 +
python/pyspark/mllib/util.py | 4 ++--
python/pyspark/sql/dataframe.py | 6 +++---
python/pyspark/sql/readwriter.py | 14 +++++++-------
python/pyspark/sql/streaming.py | 26 +++++++++++++-------------
7 files changed, 28 insertions(+), 27 deletions(-)
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index 7465cef..e1c2732 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -795,7 +795,7 @@ class BisectingKMeansModel(JavaModel,
_BisectingKMeansParams, JavaMLWritable, Ja
Computes the sum of squared distances between the input points
and their corresponding cluster centers.
- ..note:: Deprecated in 3.0.0. It will be removed in future versions.
Use
+ .. note:: Deprecated in 3.0.0. It will be removed in future versions.
Use
ClusteringEvaluator instead. You can also get the cost on the
training dataset in the
summary.
"""
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index a4c9782..f367bb8 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -53,7 +53,7 @@ class JavaRegressor(JavaPredictor, _JavaPredictorParams):
class JavaRegressionModel(JavaPredictionModel, _JavaPredictorParams):
"""
Java Model produced by a ``_JavaRegressor``.
- To be mixed in with class:`pyspark.ml.JavaModel`
+ To be mixed in with :class:`pyspark.ml.JavaModel`
.. versionadded:: 3.0.0
"""
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index 35ad551..aac2b38 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -563,6 +563,7 @@ class DefaultParamsReader(MLReader):
class HasTrainingSummary(object):
"""
Base class for models that provides Training summary.
+
.. versionadded:: 3.0.0
"""
diff --git a/python/pyspark/mllib/util.py b/python/pyspark/mllib/util.py
index 1a0ce42..f0f9cda 100644
--- a/python/pyspark/mllib/util.py
+++ b/python/pyspark/mllib/util.py
@@ -372,7 +372,7 @@ class Saveable(object):
* human-readable (JSON) model metadata to path/metadata/
* Parquet formatted data to path/data/
- The model may be loaded using py:meth:`Loader.load`.
+ The model may be loaded using :py:meth:`Loader.load`.
:param sc: Spark context used to save model data.
:param path: Path specifying the directory in which to save
@@ -412,7 +412,7 @@ class Loader(object):
def load(cls, sc, path):
"""
Load a model from the given path. The model should have been
- saved using py:meth:`Saveable.save`.
+ saved using :py:meth:`Saveable.save`.
:param sc: Spark context used for loading model files.
:param path: Path specifying the directory to which the model
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 2a366dc..6f4fdd3 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -2138,7 +2138,7 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
@ignore_unicode_prefix
def toDF(self, *cols):
- """Returns a new class:`DataFrame` that with new specified column names
+ """Returns a new :class:`DataFrame` that with new specified column
names
:param cols: list of new column names (string)
@@ -2150,9 +2150,9 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
@since(3.0)
def transform(self, func):
- """Returns a new class:`DataFrame`. Concise syntax for chaining custom
transformations.
+ """Returns a new :class:`DataFrame`. Concise syntax for chaining
custom transformations.
- :param func: a function that takes and returns a class:`DataFrame`.
+ :param func: a function that takes and returns a :class:`DataFrame`.
>>> from pyspark.sql.functions import col
>>> df = spark.createDataFrame([(1, 1.0), (2, 2.0)], ["int", "float"])
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 6ad6377..336345e 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -223,15 +223,15 @@ class DataFrameReader(OptionUtils):
:param mode: allows a mode for dealing with corrupt records during
parsing. If None is
set, it uses the default value, ``PERMISSIVE``.
- * ``PERMISSIVE`` : when it meets a corrupted record, puts the
malformed string \
+ * ``PERMISSIVE``: when it meets a corrupted record, puts the
malformed string \
into a field configured by ``columnNameOfCorruptRecord``,
and sets malformed \
fields to ``null``. To keep corrupt records, an user can set
a string type \
field named ``columnNameOfCorruptRecord`` in an user-defined
schema. If a \
schema does not have the field, it drops corrupt records
during parsing. \
When inferring a schema, it implicitly adds a
``columnNameOfCorruptRecord`` \
field in an output schema.
- * ``DROPMALFORMED`` : ignores the whole corrupted records.
- * ``FAILFAST`` : throws an exception when it meets corrupted
records.
+ * ``DROPMALFORMED``: ignores the whole corrupted records.
+ * ``FAILFAST``: throws an exception when it meets corrupted
records.
:param columnNameOfCorruptRecord: allows renaming the new field having
malformed string
created by ``PERMISSIVE`` mode. This
overrides
@@ -470,7 +470,7 @@ class DataFrameReader(OptionUtils):
be controlled by
``spark.sql.csv.parser.columnPruning.enabled``
(enabled by default).
- * ``PERMISSIVE`` : when it meets a corrupted record, puts the
malformed string \
+ * ``PERMISSIVE``: when it meets a corrupted record, puts the
malformed string \
into a field configured by ``columnNameOfCorruptRecord``,
and sets malformed \
fields to ``null``. To keep corrupt records, an user can set
a string type \
field named ``columnNameOfCorruptRecord`` in an user-defined
schema. If a \
@@ -479,8 +479,8 @@ class DataFrameReader(OptionUtils):
When it meets a record having fewer tokens than the length
of the schema, \
sets ``null`` to extra fields. When the record has more
tokens than the \
length of the schema, it drops extra tokens.
- * ``DROPMALFORMED`` : ignores the whole corrupted records.
- * ``FAILFAST`` : throws an exception when it meets corrupted
records.
+ * ``DROPMALFORMED``: ignores the whole corrupted records.
+ * ``FAILFAST``: throws an exception when it meets corrupted
records.
:param columnNameOfCorruptRecord: allows renaming the new field having
malformed string
created by ``PERMISSIVE`` mode. This
overrides
@@ -830,7 +830,7 @@ class DataFrameWriter(OptionUtils):
def insertInto(self, tableName, overwrite=None):
"""Inserts the content of the :class:`DataFrame` to the specified
table.
- It requires that the schema of the class:`DataFrame` is the same as the
+ It requires that the schema of the :class:`DataFrame` is the same as
the
schema of the table.
Optionally overwriting any existing data.
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index 05cf331..2450a4c 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -461,15 +461,15 @@ class DataStreamReader(OptionUtils):
:param mode: allows a mode for dealing with corrupt records during
parsing. If None is
set, it uses the default value, ``PERMISSIVE``.
- * ``PERMISSIVE`` : when it meets a corrupted record, puts the
malformed string \
+ * ``PERMISSIVE``: when it meets a corrupted record, puts the
malformed string \
into a field configured by ``columnNameOfCorruptRecord``,
and sets malformed \
fields to ``null``. To keep corrupt records, an user can set
a string type \
field named ``columnNameOfCorruptRecord`` in an user-defined
schema. If a \
schema does not have the field, it drops corrupt records
during parsing. \
When inferring a schema, it implicitly adds a
``columnNameOfCorruptRecord`` \
field in an output schema.
- * ``DROPMALFORMED`` : ignores the whole corrupted records.
- * ``FAILFAST`` : throws an exception when it meets corrupted
records.
+ * ``DROPMALFORMED``: ignores the whole corrupted records.
+ * ``FAILFAST``: throws an exception when it meets corrupted
records.
:param columnNameOfCorruptRecord: allows renaming the new field having
malformed string
created by ``PERMISSIVE`` mode. This
overrides
@@ -707,7 +707,7 @@ class DataStreamReader(OptionUtils):
:param mode: allows a mode for dealing with corrupt records during
parsing. If None is
set, it uses the default value, ``PERMISSIVE``.
- * ``PERMISSIVE`` : when it meets a corrupted record, puts the
malformed string \
+ * ``PERMISSIVE``: when it meets a corrupted record, puts the
malformed string \
into a field configured by ``columnNameOfCorruptRecord``,
and sets malformed \
fields to ``null``. To keep corrupt records, an user can set
a string type \
field named ``columnNameOfCorruptRecord`` in an user-defined
schema. If a \
@@ -716,8 +716,8 @@ class DataStreamReader(OptionUtils):
When it meets a record having fewer tokens than the length
of the schema, \
sets ``null`` to extra fields. When the record has more
tokens than the \
length of the schema, it drops extra tokens.
- * ``DROPMALFORMED`` : ignores the whole corrupted records.
- * ``FAILFAST`` : throws an exception when it meets corrupted
records.
+ * ``DROPMALFORMED``: ignores the whole corrupted records.
+ * ``FAILFAST``: throws an exception when it meets corrupted
records.
:param columnNameOfCorruptRecord: allows renaming the new field having
malformed string
created by ``PERMISSIVE`` mode. This
overrides
@@ -795,11 +795,11 @@ class DataStreamWriter(object):
Options include:
- * `append`:Only the new rows in the streaming DataFrame/Dataset will
be written to
+ * `append`: Only the new rows in the streaming DataFrame/Dataset will
be written to
the sink
- * `complete`:All the rows in the streaming DataFrame/Dataset will be
written to the sink
+ * `complete`: All the rows in the streaming DataFrame/Dataset will be
written to the sink
every time these is some updates
- * `update`:only the rows that were updated in the streaming
DataFrame/Dataset will be
+ * `update`: only the rows that were updated in the streaming
DataFrame/Dataset will be
written to the sink every time there are some updates. If the query
doesn't contain
aggregations, it will be equivalent to `append` mode.
@@ -1170,11 +1170,11 @@ class DataStreamWriter(object):
:param outputMode: specifies how data of a streaming DataFrame/Dataset
is written to a
streaming sink.
- * `append`:Only the new rows in the streaming DataFrame/Dataset
will be written to the
+ * `append`: Only the new rows in the streaming DataFrame/Dataset
will be written to the
sink
- * `complete`:All the rows in the streaming DataFrame/Dataset will
be written to the sink
- every time these is some updates
- * `update`:only the rows that were updated in the streaming
DataFrame/Dataset will be
+ * `complete`: All the rows in the streaming DataFrame/Dataset will
be written to the
+ sink every time these is some updates
+ * `update`: only the rows that were updated in the streaming
DataFrame/Dataset will be
written to the sink every time there are some updates. If the
query doesn't contain
aggregations, it will be equivalent to `append` mode.
:param partitionBy: names of partitioning columns
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
