This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3d292dc7b1c5 [SPARK-54028][SQL] Use empty schema when altering a view
which is not Hive compatible
3d292dc7b1c5 is described below
commit 3d292dc7b1c5b5ff977c178a88f8ee73deaee88f
Author: Chiran Ravani <[email protected]>
AuthorDate: Sat Nov 1 03:06:32 2025 +0800
[SPARK-54028][SQL] Use empty schema when altering a view which is not Hive
compatible
### What changes were proposed in this pull request?
Spark attempts to save views in a Hive-compatible format and only sets the
schema to empty if the save operation fails.
However, due to certain Hive compatibility issues, the save operation may
succeed while subsequent read operations fail. This issue arises after the
change introduced in
[SPARK-46934](https://issues.apache.org/jira/browse/SPARK-46934), which removed
the verifyColumnDataType check during the ALTER TABLE operation.
### Why are the changes needed?
to not save malformed views that no one can read.
### Does this PR introduce _any_ user-facing change?
Yes, the malformed view will be saved in non hive compatible way so that
Spark can read it.
### How was this patch tested?
Updated Test case
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #52730 from cravani/SPARK-54028.
Authored-by: Chiran Ravani <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../spark/sql/hive/HiveExternalCatalog.scala | 12 ++++++-
.../spark/sql/hive/HiveMetastoreCatalogSuite.scala | 37 ++++++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 6389e130b6a2..21bdd28dfd88 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -598,7 +598,17 @@ private[spark] class HiveExternalCatalog(conf: SparkConf,
hadoopConf: Configurat
if (tableDefinition.tableType == VIEW) {
val newTableProps = tableDefinition.properties ++
tableMetaToTableProps(tableDefinition).toMap
- val newTable = tableDefinition.copy(properties = newTableProps)
+ val schemaWithNoCollation = removeCollation(tableDefinition.schema)
+ val hiveCompatibleSchema =
+ // Spark-created views do not have to be Hive compatible. If the data
type is not
+ // Hive compatible, we can set schema to empty so that Spark can still
read this
+ // view as the schema is also encoded in the table properties.
+ if (schemaWithNoCollation.exists(f =>
!isHiveCompatibleDataType(f.dataType))) {
+ EMPTY_DATA_SCHEMA
+ } else {
+ schemaWithNoCollation
+ }
+ val newTable = tableDefinition.copy(schema = hiveCompatibleSchema,
properties = newTableProps)
try {
client.alterTable(newTable)
} catch {
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index fad374827418..a7d43ebbef07 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -438,6 +438,43 @@ class DataSourceWithHiveMetastoreCatalogSuite
}
}
+ test("SPARK-54028: Table and View with complex nested schema and ALTER
operations") {
+ withTable("t") {
+ val schema =
+ "struct_field STRUCT<" +
+ "`colon:field_name`:STRING" +
+ ">"
+ sql("CREATE TABLE t (" + schema + ")")
+
+ // Verify initial table schema
+ assert(spark.table("t").schema ===
CatalystSqlParser.parseTableSchema(schema))
+
+ withView("v") {
+ sql("CREATE VIEW v AS SELECT `struct_field` FROM t")
+
+ // Verify view schema matches the original schema
+ val expectedViewSchema = CatalystSqlParser.parseTableSchema(schema)
+ assert(spark.table("v").schema === expectedViewSchema)
+
+ // Add new column to table
+ sql("ALTER TABLE t ADD COLUMN (field_1 INT)")
+
+ // Update schema string to include new column
+ val updatedSchema = schema + ",field_1 INT"
+
+ // Verify table schema after ALTER
+ assert(spark.table("t").schema ===
CatalystSqlParser.parseTableSchema(updatedSchema))
+
+ // Alter view to include new column
+ sql("ALTER VIEW v AS " +
+ "SELECT `struct_field`,`field_1` FROM t")
+
+ // Verify view schema after ALTER
+ assert(spark.table("v").schema ===
CatalystSqlParser.parseTableSchema(updatedSchema))
+ }
+ }
+ }
+
test("SPARK-46934: Handle special characters in struct types with CTAS") {
withTable("t") {
val schema = "`a.b` struct<`a.b.b`:array<string>, `a b c`:map<int,
string>>"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
