This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 588188f481d [SPARK-43834][SQL] Use error classes in the compilation
errors of `ResolveDefaultColumns`
588188f481d is described below
commit 588188f481db899317bdc398438d6bd749224f9f
Author: panbingkun <[email protected]>
AuthorDate: Sun May 28 19:08:25 2023 +0300
[SPARK-43834][SQL] Use error classes in the compilation errors of
`ResolveDefaultColumns`
### What changes were proposed in this pull request?
The pr aims to use error classes in the compilation errors of
`ResolveDefaultColumns`.
### Why are the changes needed?
The changes improve the error framework.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- Update UT.
- Pass GA.
Closes #41345 from panbingkun/SPARK-43834.
Authored-by: panbingkun <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
---
core/src/main/resources/error/error-classes.json | 27 ++-
.../catalyst/util/ResolveDefaultColumnsUtil.scala | 21 +--
.../spark/sql/errors/QueryCompilationErrors.scala | 43 ++++-
.../sql/catalyst/catalog/SessionCatalogSuite.scala | 38 +++-
.../analysis/ResolveDefaultColumnsSuite.scala | 53 +++++-
.../org/apache/spark/sql/sources/InsertSuite.scala | 206 +++++++++++++++------
6 files changed, 290 insertions(+), 98 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json
b/core/src/main/resources/error/error-classes.json
index c8e11e6e55e..36125d2cbae 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -948,6 +948,28 @@
],
"sqlState" : "42000"
},
+ "INVALID_DEFAULT_VALUE" : {
+ "message" : [
+ "Failed to execute <statement> command because the destination table
column <colName> has a DEFAULT value <defaultValue>,"
+ ],
+ "subClass" : {
+ "DATA_TYPE" : {
+ "message" : [
+ "which requires <expectedType> type, but the statement provided a
value of incompatible <actualType> type."
+ ]
+ },
+ "SUBQUERY_EXPRESSION" : {
+ "message" : [
+ "which contains subquery expressions."
+ ]
+ },
+ "UNRESOLVED_EXPRESSION" : {
+ "message" : [
+ "which fails to resolve as a valid expression."
+ ]
+ }
+ }
+ },
"INVALID_DRIVER_MEMORY" : {
"message" : [
"System memory <systemMemory> must be at least <minSystemMemory>. Please
increase heap size using the --driver-memory option or \"<config>\" in Spark
configuration."
@@ -4048,11 +4070,6 @@
"Failed to execute <statementType> command because DEFAULT values are
not supported when adding new columns to previously existing target data source
with table provider: \"<dataSource>\"."
]
},
- "_LEGACY_ERROR_TEMP_1347" : {
- "message" : [
- "Failed to execute command because subquery expressions are not allowed
in DEFAULT values."
- ]
- },
"_LEGACY_ERROR_TEMP_2000" : {
"message" : [
"<message>. If necessary set <ansiConfig> to false to bypass this error."
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
index 8c7e2ad4f1d..0f5c413ed78 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
@@ -188,14 +188,13 @@ object ResolveDefaultColumns {
parser.parseExpression(defaultSQL)
} catch {
case ex: ParseException =>
- throw new AnalysisException(
- s"Failed to execute $statementType command because the destination
table column " +
- s"$colName has a DEFAULT value of $defaultSQL which fails to parse
as a valid " +
- s"expression: ${ex.getMessage}")
+ throw QueryCompilationErrors.defaultValuesUnresolvedExprError(
+ statementType, colName, defaultSQL, ex)
}
// Check invariants before moving on to analysis.
if (parsed.containsPattern(PLAN_EXPRESSION)) {
- throw
QueryCompilationErrors.defaultValuesMayNotContainSubQueryExpressions()
+ throw
QueryCompilationErrors.defaultValuesMayNotContainSubQueryExpressions(
+ statementType, colName, defaultSQL)
}
// Analyze the parse result.
val plan = try {
@@ -205,10 +204,8 @@ object ResolveDefaultColumns {
ConstantFolding(analyzed)
} catch {
case ex: AnalysisException =>
- throw new AnalysisException(
- s"Failed to execute $statementType command because the destination
table column " +
- s"$colName has a DEFAULT value of $defaultSQL which fails to
resolve as a valid " +
- s"expression: ${ex.getMessage}")
+ throw QueryCompilationErrors.defaultValuesUnresolvedExprError(
+ statementType, colName, defaultSQL, ex)
}
val analyzed: Expression = plan.collectFirst {
case Project(Seq(a: Alias), OneRowRelation()) => a.child
@@ -241,10 +238,8 @@ object ResolveDefaultColumns {
}
} else None
result.getOrElse {
- throw new AnalysisException(
- s"Failed to execute $statementType command because the destination
table column " +
- s"$colName has a DEFAULT value with type $dataType, but the " +
- s"statement provided a value of incompatible type
${analyzed.dataType}")
+ throw QueryCompilationErrors.defaultValuesDataTypeError(
+ statementType, colName, defaultSQL, dataType, analyzed.dataType)
}
}
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index 18ace731dd4..05b829838aa 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -3229,10 +3229,47 @@ private[sql] object QueryCompilationErrors extends
QueryErrorsBase {
"dataSource" -> dataSource))
}
- def defaultValuesMayNotContainSubQueryExpressions(): Throwable = {
+ def defaultValuesDataTypeError(
+ statement: String,
+ colName: String,
+ defaultValue: String,
+ expectedType: DataType,
+ actualType: DataType): Throwable = {
new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1347",
- messageParameters = Map.empty)
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ messageParameters = Map(
+ "statement" -> toSQLStmt(statement),
+ "colName" -> toSQLId(colName),
+ "defaultValue" -> defaultValue,
+ "actualType" -> toSQLType(actualType),
+ "expectedType" -> toSQLType(expectedType)))
+ }
+
+ def defaultValuesUnresolvedExprError(
+ statement: String,
+ colName: String,
+ defaultValue: String,
+ cause: Throwable): Throwable = {
+ new AnalysisException(
+ errorClass = "INVALID_DEFAULT_VALUE.UNRESOLVED_EXPRESSION",
+ messageParameters = Map(
+ "statement" -> toSQLStmt(statement),
+ "colName" -> toSQLId(colName),
+ "defaultValue" -> defaultValue
+ ),
+ cause = Option(cause))
+ }
+
+ def defaultValuesMayNotContainSubQueryExpressions(
+ statement: String,
+ colName: String,
+ defaultValue: String): Throwable = {
+ new AnalysisException(
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ messageParameters = Map(
+ "statement" -> toSQLStmt(statement),
+ "colName" -> toSQLId(colName),
+ "defaultValue" -> defaultValue))
}
def nullableColumnOrFieldError(name: Seq[String]): Throwable = {
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
index 9959dbf6516..f4bd884cee6 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
@@ -163,15 +163,35 @@ abstract class SessionCatalogSuite extends AnalysisTest
with Eventually {
.analyze(columnA, statementType,
ResolveDefaultColumns.EXISTS_DEFAULT_COLUMN_METADATA_KEY)
.sql == "41")
assert(ResolveDefaultColumns.analyze(columnB, statementType).sql ==
"'abc'")
- assert(intercept[AnalysisException] {
- ResolveDefaultColumns.analyze(columnC, statementType)
- }.getMessage.contains("fails to parse as a valid expression"))
- assert(intercept[AnalysisException] {
- ResolveDefaultColumns.analyze(columnD, statementType)
- }.getMessage.contains("subquery expressions are not allowed in DEFAULT
values"))
- assert(intercept[AnalysisException] {
- ResolveDefaultColumns.analyze(columnE, statementType)
- }.getMessage.contains("statement provided a value of incompatible type"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ ResolveDefaultColumns.analyze(columnC, statementType)
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.UNRESOLVED_EXPRESSION",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`c`",
+ "defaultValue" -> "_@#$%"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ ResolveDefaultColumns.analyze(columnD, statementType)
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`d`",
+ "defaultValue" -> "(select min(x) from badtable)"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ ResolveDefaultColumns.analyze(columnE, statementType)
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`e`",
+ "expectedType" -> "\"BOOLEAN\"",
+ "defaultValue" -> "41 + 1",
+ "actualType" -> "\"INT\""))
// Make sure that constant-folding default values does not take place
when the feature is
// disabled.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveDefaultColumnsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveDefaultColumnsSuite.scala
index ba52ac995b7..b21b490ea08 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveDefaultColumnsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveDefaultColumnsSuite.scala
@@ -117,15 +117,50 @@ class ResolveDefaultColumnsSuite extends QueryTest with
SharedSparkSession {
sql("select 42, timestamp_ntz'2022-01-02', date'2022-01-03', 0f"))
// If the provided default value is a literal of a different type than
the target column
// such that no coercion is possible, throw an error.
- Seq(
- "create table demos.test_ts_other (a int default 'abc') using
parquet",
- "create table demos.test_ts_other (a timestamp default '2022-01-02')
using parquet",
- "create table demos.test_ts_other (a boolean default 'true') using
parquet",
- "create table demos.test_ts_other (a int default true) using parquet"
- ).foreach { command =>
- assert(intercept[AnalysisException](sql(command))
- .getMessage.contains("statement provided a value of incompatible
type"))
- }
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table demos.test_ts_other (a int default 'abc') using
parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`a`",
+ "expectedType" -> "\"INT\"",
+ "defaultValue" -> "'abc'",
+ "actualType" -> "\"STRING\""))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table demos.test_ts_other (a timestamp default
'2022-01-02') using parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`a`",
+ "expectedType" -> "\"TIMESTAMP\"",
+ "defaultValue" -> "'2022-01-02'",
+ "actualType" -> "\"STRING\""))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table demos.test_ts_other (a boolean default 'true')
using parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`a`",
+ "expectedType" -> "\"BOOLEAN\"",
+ "defaultValue" -> "'true'",
+ "actualType" -> "\"STRING\""))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table demos.test_ts_other (a int default true) using
parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`a`",
+ "expectedType" -> "\"INT\"",
+ "defaultValue" -> "true",
+ "actualType" -> "\"BOOLEAN\""))
}
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index 3c469b98918..08d9da3e9da 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -1044,7 +1044,6 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
test("SPARK-38336 INSERT INTO statements with tables with default columns:
negative tests") {
object Errors {
val COMMON_SUBSTRING = " has a DEFAULT value"
- val BAD_SUBQUERY = "subquery expressions are not allowed in DEFAULT
values"
val TARGET_TABLE_NOT_FOUND = "The table or view `t` cannot be found"
}
// The default value fails to analyze.
@@ -1055,24 +1054,43 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
}
// The default value analyzes to a table not in the catalog.
withTable("t") {
- assert(intercept[AnalysisException] {
- sql("create table t(i boolean, s bigint default (select min(x) from
badtable)) " +
- "using parquet")
- }.getMessage.contains(Errors.BAD_SUBQUERY))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table t(i boolean, s bigint default (select min(x) from
badtable)) " +
+ "using parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`s`",
+ "defaultValue" -> "(select min(x) from badtable)"))
}
// The default value parses but refers to a table from the catalog.
withTable("t", "other") {
sql("create table other(x string) using parquet")
- assert(intercept[AnalysisException] {
- sql("create table t(i boolean, s bigint default (select min(x) from
other)) using parquet")
- }.getMessage.contains(Errors.BAD_SUBQUERY))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table t(i boolean, s bigint default (select min(x) from
other)) " +
+ "using parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`s`",
+ "defaultValue" -> "(select min(x) from other)"))
}
// The default value has an explicit alias. It fails to evaluate when
inlined into the VALUES
// list at the INSERT INTO time.
withTable("t") {
- assert(intercept[AnalysisException] {
- sql("create table t(i boolean default (select false as alias), s
bigint) using parquet")
- }.getMessage.contains(Errors.BAD_SUBQUERY))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table t(i boolean default (select false as alias), s
bigint) using parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`i`",
+ "defaultValue" -> "(select false as alias)"))
}
// Explicit default values may not participate in complex expressions in
the VALUES list.
withTable("t") {
@@ -1399,7 +1417,6 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
test("SPARK-38811 INSERT INTO on columns added with ALTER TABLE ADD COLUMNS:
Negative tests") {
object Errors {
val COMMON_SUBSTRING = " has a DEFAULT value"
- val BAD_SUBQUERY = "subquery expressions are not allowed in DEFAULT
values"
}
// The default value fails to analyze.
withTable("t") {
@@ -1411,24 +1428,44 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
// The default value analyzes to a table not in the catalog.
withTable("t") {
sql("create table t(i boolean) using parquet")
- assert(intercept[AnalysisException] {
- sql("alter table t add column s bigint default (select min(x) from
badtable)")
- }.getMessage.contains(Errors.BAD_SUBQUERY))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("alter table t add column s bigint default (select min(x) from
badtable)")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "ALTER TABLE ADD COLUMNS",
+ "colName" -> "`s`",
+ "defaultValue" -> "(select min(x) from badtable)"))
}
// The default value parses but refers to a table from the catalog.
withTable("t", "other") {
sql("create table other(x string) using parquet")
sql("create table t(i boolean) using parquet")
- assert(intercept[AnalysisException] {
- sql("alter table t add column s bigint default (select min(x) from
other)")
- }.getMessage.contains(Errors.BAD_SUBQUERY))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("alter table t add column s bigint default (select min(x) from
other)")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "ALTER TABLE ADD COLUMNS",
+ "colName" -> "`s`",
+ "defaultValue" -> "(select min(x) from other)"))
}
// The default value parses but the type is not coercible.
withTable("t") {
sql("create table t(i boolean) using parquet")
- assert(intercept[AnalysisException] {
- sql("alter table t add column s bigint default false")
- }.getMessage.contains("provided a value of incompatible type"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("alter table t add column s bigint default false")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "ALTER TABLE ADD COLUMNS",
+ "colName" -> "`s`",
+ "expectedType" -> "\"BIGINT\"",
+ "defaultValue" -> "false",
+ "actualType" -> "\"BOOLEAN\""))
}
// The default value is disabled per configuration.
withTable("t") {
@@ -1477,7 +1514,6 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
test("SPARK-38838 INSERT INTO with defaults set by ALTER TABLE ALTER COLUMN:
negative tests") {
object Errors {
val COMMON_SUBSTRING = " has a DEFAULT value"
- val BAD_SUBQUERY = "subquery expressions are not allowed in DEFAULT
values"
}
val createTable = "create table t(i boolean, s bigint) using parquet"
withTable("t") {
@@ -1487,18 +1523,38 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
sql("alter table t alter column s set default badvalue")
}.getMessage.contains(Errors.COMMON_SUBSTRING))
// The default value analyzes to a table not in the catalog.
- assert(intercept[AnalysisException] {
- sql("alter table t alter column s set default (select min(x) from
badtable)")
- }.getMessage.contains(Errors.BAD_SUBQUERY))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("alter table t alter column s set default (select min(x) from
badtable)")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "ALTER TABLE ALTER COLUMN",
+ "colName" -> "`s`",
+ "defaultValue" -> "(select min(x) from badtable)"))
// The default value has an explicit alias. It fails to evaluate when
inlined into the VALUES
// list at the INSERT INTO time.
- assert(intercept[AnalysisException] {
- sql("alter table t alter column s set default (select 42 as alias)")
- }.getMessage.contains(Errors.BAD_SUBQUERY))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("alter table t alter column s set default (select 42 as alias)")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "ALTER TABLE ALTER COLUMN",
+ "colName" -> "`s`",
+ "defaultValue" -> "(select 42 as alias)"))
// The default value parses but the type is not coercible.
- assert(intercept[AnalysisException] {
- sql("alter table t alter column s set default false")
- }.getMessage.contains("provided a value of incompatible type"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("alter table t alter column s set default false")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "ALTER TABLE ALTER COLUMN",
+ "colName" -> "`s`",
+ "expectedType" -> "\"BIGINT\"",
+ "defaultValue" -> "false",
+ "actualType" -> "\"BOOLEAN\""))
// The default value is disabled per configuration.
withSQLConf(SQLConf.ENABLE_DEFAULT_COLUMNS.key -> "false") {
val sqlText = "alter table t alter column s set default 41 + 1"
@@ -1749,9 +1805,6 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
}
// Negative tests: provided array element types must match their
corresponding DEFAULT
// declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the destination
table column s " +
- "has a DEFAULT value with type"
Seq(
Config(
"parquet"),
@@ -1765,9 +1818,17 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
} else {
sql("insert into t select false")
}
- assert(intercept[AnalysisException] {
- sql("alter table t add column s array<int> default array('abc',
'def')")
- }.getMessage.contains(incompatibleDefault))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("alter table t add column s array<int> default array('abc',
'def')")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "ALTER TABLE ADD COLUMNS",
+ "colName" -> "`s`",
+ "expectedType" -> "\"ARRAY<INT>\"",
+ "defaultValue" -> "array('abc', 'def')",
+ "actualType" -> "\"ARRAY<STRING>\""))
}
}
}
@@ -1802,9 +1863,6 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
// Negative tests: provided map element types must match their
corresponding DEFAULT
// declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the destination
table column s " +
- "has a DEFAULT value with type"
Seq(
Config(
"parquet"),
@@ -1818,9 +1876,17 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
} else {
sql("insert into t select false")
}
- assert(intercept[AnalysisException] {
- sql("alter table t add column s struct<x boolean, y string> default
struct(42, 56)")
- }.getMessage.contains(incompatibleDefault))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("alter table t add column s struct<x boolean, y string>
default struct(42, 56)")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "ALTER TABLE ADD COLUMNS",
+ "colName" -> "`s`",
+ "expectedType" -> "\"STRUCT<x: BOOLEAN, y: STRING>\"",
+ "defaultValue" -> "struct(42, 56)",
+ "actualType" -> "\"STRUCT<col1: INT, col2: INT>\""))
}
}
}
@@ -1909,9 +1975,6 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
}
// Negative tests: provided map element types must match their
corresponding DEFAULT
// declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the destination
table column s " +
- "has a DEFAULT value with type"
Seq(
Config(
"parquet"),
@@ -1925,24 +1988,49 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
} else {
sql("insert into t select false")
}
- assert(intercept[AnalysisException] {
- sql("alter table t add column s map<boolean, string> default map(42,
56)")
- }.getMessage.contains(incompatibleDefault))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("alter table t add column s map<boolean, string> default
map(42, 56)")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ parameters = Map(
+ "statement" -> "ALTER TABLE ADD COLUMNS",
+ "colName" -> "`s`",
+ "expectedType" -> "\"MAP<BOOLEAN, STRING>\"",
+ "defaultValue" -> "map(42, 56)",
+ "actualType" -> "\"MAP<INT, INT>\""))
}
}
}
test("SPARK-39643 Prohibit subquery expressions in DEFAULT values") {
- Seq(
- "create table t(a string default (select 'abc')) using parquet",
- "create table t(a string default exists(select 42 where true)) using
parquet",
- "create table t(a string default 1 in (select 1 union all select 2))
using parquet"
- ).foreach { query =>
- assert(intercept[AnalysisException] {
- sql(query)
- }.getMessage.contains(
-
QueryCompilationErrors.defaultValuesMayNotContainSubQueryExpressions().getMessage))
- }
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table t(a string default (select 'abc')) using parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`a`",
+ "defaultValue" -> "(select 'abc')"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table t(a string default exists(select 42 where true))
using parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`a`",
+ "defaultValue" -> "exists(select 42 where true)"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("create table t(a string default 1 in (select 1 union all select
2)) using parquet")
+ },
+ errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map(
+ "statement" -> "CREATE TABLE",
+ "colName" -> "`a`",
+ "defaultValue" -> "1 in (select 1 union all select 2)"))
}
test("SPARK-39844 Restrict adding DEFAULT columns for existing tables to
certain sources") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
