This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 01edac40ec85 [SPARK-51219][SQL] Fix `ShowTablesExec.isTempView` to
work with non-`V2SessionCatalog` catalogs
01edac40ec85 is described below
commit 01edac40ec858a95c7c51419aa3b12f28a9595de
Author: Dima <[email protected]>
AuthorDate: Tue Feb 18 20:46:08 2025 +0800
[SPARK-51219][SQL] Fix `ShowTablesExec.isTempView` to work with
non-`V2SessionCatalog` catalogs
### What changes were proposed in this pull request?
When non buildin catalog is configured (for example,
`org.apache.spark.sql.delta.catalog.DeltaCatalog`) and there is a temp table in
catalog, running catalog listTable will fail:
```scala
spark.conf.set("spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.delta.catalog.DeltaCatalog")
spark.range(0,2).createOrReplaceTempView("abc")
spark.catalog.listTables().show()
// org.apache.spark.sql.catalyst.parser.ParseException:
// [PARSE_EMPTY_STATEMENT] Syntax error, unexpected empty statement.
SQLSTATE: 42617 (line 1, pos 0)
//
// == SQL ==
//
// ^^^
```
If default `V2SessionCatalog` catalog is in use, or there are no temp
tables, the same command run without issues.
This behavior is due to `ShowTablesExec. isTempView ` method, where only
for `V2SessionCatalog` catalogs dedicated `isTempView` is executed. This PR
fixes that, by using `session.sessionState.catalog.isTempView` instead.
### Why are the changes needed?
To avoid unnecessary fails when a non-buildin v2 catalog is in use.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New unit tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #49959 from ostronaut/features/fix-ShowTablesExec-isTempView.
Authored-by: Dima <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit aa37f89811878d9993962c75b73025bcadcbb6f5)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../sql/execution/datasources/v2/ShowTablesExec.scala | 15 ++++++++-------
.../org/apache/spark/sql/internal/CatalogSuite.scala | 12 +++++++++++-
2 files changed, 19 insertions(+), 8 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
index cde3dfb7ead5..f314dfd79119 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
@@ -22,9 +22,10 @@ import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.util.StringUtils
-import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
+import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier,
TableCatalog}
import
org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
import org.apache.spark.sql.execution.LeafExecNode
+import org.apache.spark.util.ArrayImplicits._
/**
* Physical plan node for showing tables.
@@ -40,17 +41,17 @@ case class ShowTablesExec(
val tables = catalog.listTables(namespace.toArray)
tables.map { table =>
if (pattern.map(StringUtils.filterPattern(Seq(table.name()),
_).nonEmpty).getOrElse(true)) {
- rows += toCatalystRow(table.namespace().quoted, table.name(),
isTempView(table))
+ rows += toCatalystRow(table.namespace().quoted, table.name(),
isTempView(table, catalog))
}
}
rows.toSeq
}
- private def isTempView(ident: Identifier): Boolean = {
- catalog match {
- case s: V2SessionCatalog => s.isTempView(ident)
- case _ => false
- }
+ private def isTempView(ident: Identifier, catalog: TableCatalog): Boolean = {
+ if (CatalogV2Util.isSessionCatalog(catalog)) {
+ session.sessionState.catalog
+ .isTempView((ident.namespace() :+ ident.name()).toImmutableArraySeq)
+ } else false
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
index cd52b52a3e1f..1fe7530044e5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
import org.apache.spark.sql.catalyst.plans.logical.Range
import org.apache.spark.sql.classic.Catalog
-import org.apache.spark.sql.connector.FakeV2Provider
+import org.apache.spark.sql.connector.{FakeV2Provider,
InMemoryTableSessionCatalog}
import org.apache.spark.sql.connector.catalog.{CatalogManager, Identifier,
InMemoryCatalog}
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.CatalogHelper
import org.apache.spark.sql.connector.catalog.functions._
@@ -271,6 +271,16 @@ class CatalogSuite extends SharedSparkSession with
AnalysisTest with BeforeAndAf
Set("testcat.my_db.my_table2"))
}
+ test("SPARK-51219: list tables with non-buildin V2 catalog") {
+ withSQLConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key ->
+ classOf[InMemoryTableSessionCatalog].getName) {
+ createTable("my_table")
+ createTempTable("my_temp_table")
+ assert(spark.catalog.listTables().collect().map(_.name).toSet ==
+ Set("my_table", "my_temp_table"))
+ }
+ }
+
test("list tables with database") {
assert(spark.catalog.listTables("default").collect().isEmpty)
createDatabase("my_db1")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
