This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 878dd6abce76 [SPARK-48601][SQL] Give a more user friendly error
message when setting a null value for JDBC Option
878dd6abce76 is described below
commit 878dd6abce767de0981074d61335662d747aedbf
Author: Stevo Mitric <[email protected]>
AuthorDate: Wed Jun 19 09:58:05 2024 +0800
[SPARK-48601][SQL] Give a more user friendly error message when setting a
null value for JDBC Option
### What changes were proposed in this pull request?
In this PR, proposed changes add a check for validating that JDBC Option
values are not null, and throw a user-friendly error in case that they are.
### Why are the changes needed?
When setting a `null` value for JDBC Option, a spark internal exception is
thrown due to java.lang.NullPointerException. A more user-friendly message
should be thrown in such cases.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
A new test in this PR.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #46955 from stevomitric/stevomitric/fix-jdbcoptions.
Authored-by: Stevo Mitric <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../utils/src/main/resources/error/error-conditions.json | 6 ++++++
.../apache/spark/sql/errors/QueryCompilationErrors.scala | 7 +++++++
.../spark/sql/execution/datasources/jdbc/JDBCOptions.scala | 11 +++++++++--
.../test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala | 14 ++++++++++++++
4 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json
b/common/utils/src/main/resources/error/error-conditions.json
index 35dfa7a6c349..9a89fa29858d 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -3328,6 +3328,12 @@
],
"sqlState" : "42000"
},
+ "NULL_DATA_SOURCE_OPTION" : {
+ "message" : [
+ "Data source read/write option <option> cannot have null value."
+ ],
+ "sqlState" : "22024"
+ },
"NULL_MAP_KEY" : {
"message" : [
"Cannot use null as map key."
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index 7b9eb2020a5f..18a1f7dffd6b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -175,6 +175,13 @@ private[sql] object QueryCompilationErrors extends
QueryErrorsBase with Compilat
"functionName" -> toSQLId(funcName)))
}
+ def nullDataSourceOption(option: String): Throwable = {
+ new AnalysisException(
+ errorClass = "NULL_DATA_SOURCE_OPTION",
+ messageParameters = Map("option" -> option)
+ )
+ }
+
def unorderablePivotColError(pivotCol: Expression): Throwable = {
new AnalysisException(
errorClass = "INCOMPARABLE_PIVOT_COLUMN",
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
index 43db0c6eef11..481cc80fe522 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala
@@ -25,7 +25,7 @@ import org.apache.commons.io.FilenameUtils
import org.apache.spark.SparkFiles
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
-import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.errors.{QueryCompilationErrors,
QueryExecutionErrors}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.TimestampNTZType
import org.apache.spark.util.Utils
@@ -52,7 +52,14 @@ class JDBCOptions(
*/
val asProperties: Properties = {
val properties = new Properties()
- parameters.originalMap.foreach { case (k, v) => properties.setProperty(k,
v) }
+ parameters.originalMap.foreach { case (k, v) =>
+ // If an option value is `null`, throw a user-friendly error. Keys here
cannot be null, as
+ // scala's implementation of Maps prohibits null keys.
+ if (v == null) {
+ throw QueryCompilationErrors.nullDataSourceOption(k)
+ }
+ properties.setProperty(k, v)
+ }
properties
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
index 8e98181a9802..e1a7971b283c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
@@ -369,6 +369,20 @@ class JDBCV2Suite extends QueryTest with
SharedSparkSession with ExplainSuiteHel
}
}
+ test("null value for option exception") {
+ val df = spark.read
+ .option("pushDownOffset", null)
+ .table("h2.test.employee")
+ checkError(
+ exception = intercept[AnalysisException] {
+ df.collect()
+ },
+ errorClass = "NULL_DATA_SOURCE_OPTION",
+ parameters = Map(
+ "option" -> "pushDownOffset")
+ )
+ }
+
test("simple scan with OFFSET") {
val df1 = spark.read
.table("h2.test.employee")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
