Repository: spark Updated Branches: refs/heads/branch-2.0 43bd612f3 -> 5626a0af5
[SPARK-16236][SQL] Add Path Option back to Load API in DataFrameReader #### What changes were proposed in this pull request? koertkuipers identified the PR https://github.com/apache/spark/pull/13727/ changed the behavior of `load` API. After the change, the `load` API does not add the value of `path` into the `options`. Thank you! This PR is to add the option `path` back to `load()` API in `DataFrameReader`, if and only if users specify one and only one `path` in the `load` API. For example, users can see the `path` option after the following API call, ```Scala spark.read .format("parquet") .load("/test") ``` #### How was this patch tested? Added test cases. Author: gatorsmile <[email protected]> Closes #13933 from gatorsmile/optionPath. (cherry picked from commit 25520e976275e0d1e3bf9c73128ef4dec4618568) Signed-off-by: Reynold Xin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5626a0af Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5626a0af Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5626a0af Branch: refs/heads/branch-2.0 Commit: 5626a0af598168a15d68a8817d1dec2a0e3dec7e Parents: 43bd612 Author: gatorsmile <[email protected]> Authored: Tue Jun 28 15:32:45 2016 -0700 Committer: Reynold Xin <[email protected]> Committed: Tue Jun 28 15:32:52 2016 -0700 ---------------------------------------------------------------------- .../org/apache/spark/sql/DataFrameReader.scala | 2 +- .../sql/test/DataFrameReaderWriterSuite.scala | 28 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/5626a0af/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala index 35ba9c5..35ba522 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala @@ -129,7 +129,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { * @since 1.4.0 */ def load(path: String): DataFrame = { - load(Seq(path): _*) // force invocation of `load(...varargs...)` + option("path", path).load(Seq.empty: _*) // force invocation of `load(...varargs...)` } /** http://git-wip-us.apache.org/repos/asf/spark/blob/5626a0af/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala index 3fa3864..ebbcc1d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala @@ -170,6 +170,34 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSQLContext with Be assert(LastOptions.saveMode === SaveMode.ErrorIfExists) } + test("test path option in load") { + spark.read + .format("org.apache.spark.sql.test") + .option("intOpt", 56) + .load("/test") + + assert(LastOptions.parameters("intOpt") == "56") + assert(LastOptions.parameters("path") == "/test") + + LastOptions.clear() + spark.read + .format("org.apache.spark.sql.test") + .option("intOpt", 55) + .load() + + assert(LastOptions.parameters("intOpt") == "55") + assert(!LastOptions.parameters.contains("path")) + + LastOptions.clear() + spark.read + .format("org.apache.spark.sql.test") + .option("intOpt", 54) + .load("/test", "/test1", "/test2") + + assert(LastOptions.parameters("intOpt") == "54") + assert(!LastOptions.parameters.contains("path")) + } + test("test different data types for options") { val df = spark.read .format("org.apache.spark.sql.test") --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
