This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 89cdd638b0e2 [SPARK-54761][SQL] Throw unsupportedTableOperation for
constraint operation on DSv1/HMS table
89cdd638b0e2 is described below
commit 89cdd638b0e282ad1890431e1d60059180efc08f
Author: yhuang-db <[email protected]>
AuthorDate: Fri Dec 19 14:26:11 2025 -0800
[SPARK-54761][SQL] Throw unsupportedTableOperation for constraint operation
on DSv1/HMS table
### What changes were proposed in this pull request?
This PR throws unsupportedTableOperation for constraint operations on
DSv1/HMS table. The operations include:
- CreateTable/CreateTableAsSelect with constraint
- AddConstraint
- DropConstraint
- AddCheckConstraint
### Why are the changes needed?
Constraints (DSv2) are not supported on DSv1/HMS tables. Currently, it does
not throw exceptions and causes
[confusion](https://github.com/apache/spark/pull/50761#issuecomment-3664237003).
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New unit tests on dsv1 and hive table.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: claude-4.5-opus
Closes #53532 from
yhuang-db/SPARK-54761_unsupportedTableOperation-for-constraint-on-dsv1-hms.
Authored-by: yhuang-db <[email protected]>
Signed-off-by: Gengliang Wang <[email protected]>
(cherry picked from commit 4b90a265c6234570d29d5f4482ddd955107174de)
Signed-off-by: Gengliang Wang <[email protected]>
---
.../catalyst/analysis/ResolveSessionCatalog.scala | 31 +++++-
.../command/v1/TableConstraintSuite.scala | 121 +++++++++++++++++++++
.../command/v2/CheckConstraintSuite.scala | 58 +++++-----
.../execution/command/v2/DescribeTableSuite.scala | 8 +-
.../command/v2/ShowCreateTableSuite.scala | 10 +-
.../execution/command/TableConstraintSuite.scala | 26 +++++
6 files changed, 217 insertions(+), 37 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index b1fd0bff1071..a15ba2796202 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -21,7 +21,7 @@ import org.apache.spark.SparkException
import org.apache.spark.internal.LogKeys.CONFIG
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
-import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat,
CatalogTable, CatalogTableType, CatalogUtils, ClusterBySpec}
+import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat,
CatalogTable, CatalogTableType, CatalogUtils, ClusterBySpec, HiveTableRelation}
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.Rule
@@ -31,7 +31,7 @@ import
org.apache.spark.sql.connector.catalog.{CatalogExtension, CatalogManager,
import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.errors.{QueryCompilationErrors,
QueryExecutionErrors}
import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.{CreateTable =>
CreateTableV1}
+import org.apache.spark.sql.execution.datasources.{CreateTable =>
CreateTableV1, LogicalRelation}
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils
import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
import org.apache.spark.sql.internal.connector.V1Function
@@ -128,6 +128,25 @@ class ResolveSessionCatalog(val catalogManager:
CatalogManager)
case DropColumns(ResolvedV1TableIdentifier(ident), _, _) =>
throw QueryCompilationErrors.unsupportedTableOperationError(ident, "DROP
COLUMN")
+ // V1 and hive tables do not support constraints
+ case AddConstraint(ResolvedV1TableIdentifier(ident), _) =>
+ throw QueryCompilationErrors.unsupportedTableOperationError(ident, "ADD
CONSTRAINT")
+
+ case DropConstraint(ResolvedV1TableIdentifier(ident), _, _, _) =>
+ throw QueryCompilationErrors.unsupportedTableOperationError(ident, "DROP
CONSTRAINT")
+
+ case a: AddCheckConstraint
+ if a.child.exists {
+ case _: LogicalRelation => true
+ case _: HiveTableRelation => true
+ case _ => false
+ } =>
+ val tableIdent = a.child.collectFirst {
+ case l: LogicalRelation => l.catalogTable.get.identifier
+ case h: HiveTableRelation => h.tableMeta.identifier
+ }.get
+ throw QueryCompilationErrors.unsupportedTableOperationError(tableIdent,
"ADD CONSTRAINT")
+
case SetTableProperties(ResolvedV1TableIdentifier(ident), props) =>
AlterTableSetPropertiesCommand(ident, props, isView = false)
@@ -187,6 +206,10 @@ class ResolveSessionCatalog(val catalogManager:
CatalogManager)
c.tableSpec.provider, tableSpec.options, c.tableSpec.location,
c.tableSpec.serde,
ctas = false)
if (!isV2Provider(provider)) {
+ if (tableSpec.constraints.nonEmpty) {
+ throw QueryCompilationErrors.unsupportedTableOperationError(
+ ident, "CONSTRAINT")
+ }
constructV1TableCmd(None, c.tableSpec, ident, c.tableSchema,
c.partitioning,
c.ignoreIfExists, storageFormat, provider)
} else {
@@ -203,6 +226,10 @@ class ResolveSessionCatalog(val catalogManager:
CatalogManager)
ctas = true)
if (!isV2Provider(provider)) {
+ if (tableSpec.constraints.nonEmpty) {
+ throw QueryCompilationErrors.unsupportedTableOperationError(
+ ident, "CONSTRAINT")
+ }
constructV1TableCmd(Some(c.query), c.tableSpec, ident, new StructType,
c.partitioning,
c.ignoreIfExists, storageFormat, provider)
} else {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/TableConstraintSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/TableConstraintSuite.scala
new file mode 100644
index 000000000000..f8b3411d59a0
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/TableConstraintSuite.scala
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.execution.command.DDLCommandTestUtils
+
+/**
+ * This base suite contains unified tests for table constraints (CHECK,
PRIMARY KEY, UNIQUE,
+ * FOREIGN KEY) that check V1 table catalogs. V1 tables do not support table
constraints.
+ * The tests that cannot run for all V1 catalogs are located in more specific
test suites:
+ *
+ * - V1 In-Memory catalog:
+ * `org.apache.spark.sql.execution.command.v1.TableConstraintSuite`
+ * - V1 Hive External catalog:
+ * `org.apache.spark.sql.hive.execution.command.TableConstraintSuite`
+ */
+trait TableConstraintSuiteBase extends DDLCommandTestUtils {
+ override val command = "TABLE CONSTRAINT"
+
+ private val constraintTypes = Seq(
+ "CHECK (id > 0)",
+ "PRIMARY KEY (id)",
+ "UNIQUE (id)",
+ "FOREIGN KEY (id) REFERENCES t2(id)"
+ )
+
+ gridTest("SPARK-54761: create table with constraint - should
fail")(constraintTypes)
+ { constraint =>
+ withNamespaceAndTable("ns", "table_1") { t =>
+ val createTableSql = s"CREATE TABLE $t (id INT, CONSTRAINT c1
$constraint) $defaultUsing"
+ val error = intercept[AnalysisException] {
+ sql(createTableSql)
+ }
+ checkError(
+ exception = error,
+ condition = "UNSUPPORTED_FEATURE.TABLE_OPERATION",
+ parameters = Map(
+ "tableName" -> s"`$catalog`.`ns`.`table_1`",
+ "operation" -> "CONSTRAINT"
+ )
+ )
+ }
+ }
+
+ gridTest("SPARK-54761: alter table add constraint - should
fail")(constraintTypes) { constraint =>
+ withNamespaceAndTable("ns", "table_1") { t =>
+ sql(s"CREATE TABLE $t (id INT) $defaultUsing")
+ val alterTableSql = s"ALTER TABLE $t ADD CONSTRAINT c1 $constraint"
+ val error = intercept[AnalysisException] {
+ sql(alterTableSql)
+ }
+ checkError(
+ exception = error,
+ condition = "UNSUPPORTED_FEATURE.TABLE_OPERATION",
+ parameters = Map(
+ "tableName" -> s"`$catalog`.`ns`.`table_1`",
+ "operation" -> "ADD CONSTRAINT"
+ )
+ )
+ }
+ }
+
+ test("SPARK-54761: alter table drop constraint - should fail") {
+ withNamespaceAndTable("ns", "table_1") { t =>
+ sql(s"CREATE TABLE $t (id INT) $defaultUsing")
+ val error = intercept[AnalysisException] {
+ sql(s"ALTER TABLE $t DROP CONSTRAINT c1")
+ }
+ checkError(
+ exception = error,
+ condition = "UNSUPPORTED_FEATURE.TABLE_OPERATION",
+ parameters = Map(
+ "tableName" -> s"`$catalog`.`ns`.`table_1`",
+ "operation" -> "DROP CONSTRAINT"
+ )
+ )
+ }
+ }
+
+ // REPLACE TABLE is not supported for V1 tables, so the error should be about
+ // REPLACE TABLE, not about CONSTRAINT
+ gridTest("SPARK-54761: replace table with constraint - should
fail")(constraintTypes)
+ { constraint =>
+ withNamespaceAndTable("ns", "table_1") { t =>
+ val replaceTableSql = s"REPLACE TABLE $t (id INT, CONSTRAINT c1
$constraint) $defaultUsing"
+ val error = intercept[AnalysisException] {
+ sql(replaceTableSql)
+ }
+ checkError(
+ exception = error,
+ condition = "UNSUPPORTED_FEATURE.TABLE_OPERATION",
+ parameters = Map(
+ "tableName" -> s"`$catalog`.`ns`.`table_1`",
+ "operation" -> "REPLACE TABLE"
+ )
+ )
+ }
+ }
+}
+
+/**
+ * The class contains tests for table constraints to check V1 In-Memory table
catalog.
+ */
+class TableConstraintSuite extends TableConstraintSuiteBase with
CommandSuiteBase
+
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
index 44c5b1ad2874..ee2dd476958e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CheckConstraintSuite.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.execution.command.v2
import org.apache.spark.SparkRuntimeException
import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.catalyst.util.AttributeNameParser
+import org.apache.spark.sql.catalyst.util.QuotingUtils.quoteNameParts
import org.apache.spark.sql.connector.catalog.Table
import org.apache.spark.sql.connector.catalog.constraints.Check
import org.apache.spark.sql.execution.command.DDLCommandTestUtils
@@ -29,11 +31,11 @@ class CheckConstraintSuite extends QueryTest with
CommandSuiteBase with DDLComma
override protected def command: String = "Check CONSTRAINT"
test("Nondeterministic expression -- alter table") {
- withTable("t") {
- sql("create table t(i double)")
+ withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
+ sql(s"CREATE TABLE $t (i DOUBLE) $defaultUsing")
val query =
- """
- |ALTER TABLE t ADD CONSTRAINT c1 CHECK (i > rand(0))
+ s"""
+ |ALTER TABLE $t ADD CONSTRAINT c1 CHECK (i > rand(0))
|""".stripMargin
val error = intercept[AnalysisException] {
sql(query)
@@ -45,8 +47,8 @@ class CheckConstraintSuite extends QueryTest with
CommandSuiteBase with DDLComma
parameters = Map("checkCondition" -> "i > rand(0)"),
context = ExpectedContext(
fragment = "i > rand(0)",
- start = 40,
- stop = 50
+ start = 67,
+ stop = 77
)
)
}
@@ -77,27 +79,31 @@ class CheckConstraintSuite extends QueryTest with
CommandSuiteBase with DDLComma
}
test("Expression referring a column of another table -- alter table") {
- withTable("t", "t2") {
- sql("CREATE TABLE t(i DOUBLE) USING parquet")
- sql("CREATE TABLE t2(j STRING) USING parquet")
- val query =
- """
- |ALTER TABLE t ADD CONSTRAINT c1 CHECK (len(t2.j) > 0)
- |""".stripMargin
- val error = intercept[AnalysisException] {
- sql(query)
- }
- checkError(
- exception = error,
- condition = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
- sqlState = "42703",
- parameters = Map("objectName" -> "`t2`.`j`", "proposal" -> "`t`.`i`"),
- context = ExpectedContext(
- fragment = "t2.j",
- start = 44,
- stop = 47
+ withNamespaceAndTable("ns", "tbl_1", nonPartitionCatalog) { t1 =>
+ withNamespaceAndTable("ns", "tbl_2", nonPartitionCatalog) { t2 =>
+ sql(s"CREATE TABLE $t1(i DOUBLE) $defaultUsing")
+ sql(s"CREATE TABLE $t2(j STRING) $defaultUsing")
+ val query =
+ s"""
+ |ALTER TABLE $t1 ADD CONSTRAINT c1 CHECK (len($t2.j) > 0)
+ |""".stripMargin
+ val error = intercept[AnalysisException] {
+ sql(query)
+ }
+ checkError(
+ exception = error,
+ condition = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
+ sqlState = "42703",
+ parameters = Map(
+ "objectName" ->
quoteNameParts(AttributeNameParser.parseAttributeName(s"$t2.j")),
+ "proposal" ->
quoteNameParts(AttributeNameParser.parseAttributeName(s"$t1.i"))),
+ context = ExpectedContext(
+ fragment = s"$t2.j",
+ start = 73,
+ stop = 104
+ )
)
- )
+ }
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala
index dd84e4d1420e..847a956c11ab 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeTableSuite.scala
@@ -217,10 +217,10 @@ class DescribeTableSuite extends
command.DescribeTableSuiteBase
test("desc table constraints") {
withNamespaceAndTable("ns", "pk_table", nonPartitionCatalog) { tbl =>
- withTable("fk_table") {
+ withNamespaceAndTable("ns", "fk_table", nonPartitionCatalog) { fkTable =>
sql(
s"""
- |CREATE TABLE fk_table (id INT PRIMARY KEY) USING parquet
+ |CREATE TABLE $fkTable (id INT PRIMARY KEY) $defaultUsing
""".stripMargin)
sql(
s"""
@@ -230,7 +230,7 @@ class DescribeTableSuite extends
command.DescribeTableSuiteBase
| b STRING,
| c STRING,
| PRIMARY KEY (id),
- | CONSTRAINT fk_a FOREIGN KEY (a) REFERENCES fk_table(id) RELY,
+ | CONSTRAINT fk_a FOREIGN KEY (a) REFERENCES $fkTable(id) RELY,
| CONSTRAINT uk_b UNIQUE (b),
| CONSTRAINT uk_a_c UNIQUE (a, c),
| CONSTRAINT c1 CHECK (c IS NOT NULL),
@@ -243,7 +243,7 @@ class DescribeTableSuite extends
command.DescribeTableSuiteBase
var expectedConstraintsDdl = Array(
"# Constraints,,",
"pk_table_pk,PRIMARY KEY (id) NOT ENFORCED,",
- "fk_a,FOREIGN KEY (a) REFERENCES fk_table (id) NOT ENFORCED RELY,",
+ s"fk_a,FOREIGN KEY (a) REFERENCES $fkTable (id) NOT ENFORCED RELY,",
"uk_b,UNIQUE (b) NOT ENFORCED,",
"uk_a_c,UNIQUE (a, c) NOT ENFORCED,",
"c1,CHECK (c IS NOT NULL) ENFORCED,",
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala
index 2e3929d906ce..a9b33584efc4 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala
@@ -184,13 +184,13 @@ class ShowCreateTableSuite extends
command.ShowCreateTableSuiteBase with Command
test("show table constraints") {
withNamespaceAndTable("ns", "tbl", nonPartitionCatalog) { t =>
- withTable("other_table") {
+ withNamespaceAndTable("ns", "other_table", nonPartitionCatalog) {
otherTable =>
sql(
s"""
- |CREATE TABLE other_table (
+ |CREATE TABLE $otherTable (
| id STRING PRIMARY KEY
|)
- |USING parquet
+ |$defaultUsing
""".stripMargin)
sql(
s"""
@@ -200,7 +200,7 @@ class ShowCreateTableSuite extends
command.ShowCreateTableSuiteBase with Command
| c STRING,
| PRIMARY KEY (a),
| CONSTRAINT uk_b UNIQUE (b),
- | CONSTRAINT fk_c FOREIGN KEY (c) REFERENCES other_table(id)
RELY,
+ | CONSTRAINT fk_c FOREIGN KEY (c) REFERENCES $otherTable(id)
RELY,
| CONSTRAINT c1 CHECK (c IS NOT NULL),
| CONSTRAINT c2 CHECK (a > 0)
|)
@@ -214,7 +214,7 @@ class ShowCreateTableSuite extends
command.ShowCreateTableSuiteBase with Command
"c STRING,",
"CONSTRAINT tbl_pk PRIMARY KEY (a) NOT ENFORCED NORELY,",
"CONSTRAINT uk_b UNIQUE (b) NOT ENFORCED NORELY,",
- "CONSTRAINT fk_c FOREIGN KEY (c) REFERENCES other_table (id) NOT
ENFORCED RELY,",
+ s"CONSTRAINT fk_c FOREIGN KEY (c) REFERENCES $otherTable (id) NOT
ENFORCED RELY,",
"CONSTRAINT c1 CHECK (c IS NOT NULL) ENFORCED NORELY,"
)
assert(showDDL === expectedDDLPrefix ++ Array(
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/TableConstraintSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/TableConstraintSuite.scala
new file mode 100644
index 000000000000..540b19e4577c
--- /dev/null
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/TableConstraintSuite.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.execution.command.v1
+
+/**
+ * The class contains tests for table constraints to check V1 Hive external
table catalog.
+ */
+class TableConstraintSuite extends v1.TableConstraintSuiteBase with
CommandSuiteBase
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
