This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new b3d6bb435c8d [SPARK-49671][SQL] Remove the RTRIM collation config
b3d6bb435c8d is described below
commit b3d6bb435c8d36a2d939df9a875823b393469c4d
Author: Uros Bojanic <[email protected]>
AuthorDate: Wed May 13 09:02:06 2026 +0800
[SPARK-49671][SQL] Remove the RTRIM collation config
### What changes were proposed in this pull request?
Cleanup SQL configs by removing `TRIM_COLLATION_ENABLED` and its
surrounding code changes, thus eliminating the
`spark.sql.collation.trim.enabled` feature flag from the codebase.
### Why are the changes needed?
Collations and RTRIM are already enabled by default, and these configs are
no longer needed.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests suffice. Outdated tests have been removed.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #53576 from uros-db/remove-rtrim-flag.
Authored-by: Uros Bojanic <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit a7c0d213480d98a5d18fa0931d85cf286a8f31d0)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../expressions/collationExpressions.scala | 5 ----
.../spark/sql/catalyst/parser/AstBuilder.scala | 6 +---
.../spark/sql/errors/QueryCompilationErrors.scala | 7 -----
.../org/apache/spark/sql/internal/SQLConf.scala | 12 --------
.../sql/errors/QueryCompilationErrorsSuite.scala | 33 ----------------------
5 files changed, 1 insertion(+), 62 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala
index 9d7c2236678d..c3db6fca6a86 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala
@@ -24,7 +24,6 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.trees.TreePattern.{TreePattern,
UNRESOLVED_COLLATION}
import org.apache.spark.sql.catalyst.util.{AttributeNameParser,
CollationFactory}
import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.types.StringTypeWithCollation
import org.apache.spark.sql.types._
@@ -54,10 +53,6 @@ object CollateExpressionBuilder extends ExpressionBuilder {
if (evalCollation == null) {
throw QueryCompilationErrors.unexpectedNullError("collation",
collationExpr)
} else {
- if (!SQLConf.get.trimCollationEnabled &&
- evalCollation.toString.toUpperCase().contains("TRIM")) {
- throw QueryCompilationErrors.trimCollationNotEnabledError()
- }
Collate(e, UnresolvedCollation(
AttributeNameParser.parseAttributeName(evalCollation.toString)))
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 95b21eb01b4b..460cf816e57c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -3549,11 +3549,7 @@ class AstBuilder extends DataTypeAstBuilder
}
override def visitCollateClause(ctx: CollateClauseContext): Seq[String] =
withOrigin(ctx) {
- val collationName = visitMultipartIdentifier(ctx.collationName)
- if (!SQLConf.get.trimCollationEnabled &&
collationName.last.toUpperCase().contains("TRIM")) {
- throw QueryCompilationErrors.trimCollationNotEnabledError()
- }
- collationName
+ visitMultipartIdentifier(ctx.collationName)
}
/**
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index e6cf2ded7622..ceb384b2f533 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -396,13 +396,6 @@ private[sql] object QueryCompilationErrors extends
QueryErrorsBase with Compilat
)
}
- def trimCollationNotEnabledError(): Throwable = {
- new AnalysisException(
- errorClass = "UNSUPPORTED_FEATURE.TRIM_COLLATION",
- messageParameters = Map.empty
- )
- }
-
def trailingCommaInSelectError(origin: Origin): Throwable = {
new AnalysisException(
errorClass = "TRAILING_COMMA_IN_SELECT",
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 185ff3c3f444..a0477be8a382 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1141,16 +1141,6 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
- val TRIM_COLLATION_ENABLED =
- buildConf("spark.sql.collation.trim.enabled")
- .internal()
- .doc("When enabled allows the use of trim collations which trim trailing
whitespaces from" +
- " strings."
- )
- .version("4.0.0")
- .booleanConf
- .createWithDefault(true)
-
val COLLATION_AWARE_HASHING_ENABLED =
buildConf("spark.sql.legacy.collationAwareHashFunctions")
.internal()
@@ -7704,8 +7694,6 @@ class SQLConf extends Serializable with Logging with
SqlApiConf {
def schemaLevelCollationsEnabled: Boolean =
getConf(SCHEMA_LEVEL_COLLATIONS_ENABLED)
- def trimCollationEnabled: Boolean = getConf(TRIM_COLLATION_ENABLED)
-
def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED)
def adaptiveExecutionEnabledInStatelessStreaming: Boolean =
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
index 2c10497c190e..c626d7183513 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
@@ -876,39 +876,6 @@ class QueryCompilationErrorsSuite
"inputTypes" -> "[\"INT\", \"STRING\", \"STRING\"]"))
}
- test("SPARK-49666: the trim collation feature is off without collate builder
call") {
- withSQLConf(SQLConf.TRIM_COLLATION_ENABLED.key -> "false") {
- Seq(
- "CREATE TABLE t(col STRING COLLATE EN_RTRIM_CI) USING parquet",
- "CREATE TABLE t(col STRING COLLATE UTF8_LCASE_RTRIM) USING parquet",
- "SELECT 'aaa' COLLATE UNICODE_LTRIM_CI"
- ).foreach { sqlText =>
- checkError(
- exception = intercept[AnalysisException](sql(sqlText)),
- condition = "UNSUPPORTED_FEATURE.TRIM_COLLATION"
- )
- }
- }
- }
-
- test("SPARK-49666: the trim collation feature is off with collate builder
call") {
- withSQLConf(SQLConf.TRIM_COLLATION_ENABLED.key -> "false") {
- Seq(
- "SELECT collate('aaa', 'UNICODE_RTRIM')",
- "SELECT collate('aaa', 'UTF8_BINARY_RTRIM')",
- "SELECT collate('aaa', 'EN_AI_RTRIM')"
- ).foreach { sqlText =>
- checkError(
- exception = intercept[AnalysisException](sql(sqlText)),
- condition = "UNSUPPORTED_FEATURE.TRIM_COLLATION",
- parameters = Map.empty,
- context =
- ExpectedContext(fragment = sqlText.substring(7), start = 7, stop =
sqlText.length - 1)
- )
- }
- }
- }
-
test("SPARK-50779: the object level collations feature is unsupported when
flag is disabled") {
withSQLConf(SQLConf.OBJECT_LEVEL_COLLATIONS_ENABLED.key -> "false") {
Seq(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
