This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new b7224a8 [SPARK-57086] Support `recoverPartitions` in `Catalog`
b7224a8 is described below
commit b7224a88e763f76a7e2709f9ea36b496fa438fef
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue May 26 14:58:09 2026 -0700
[SPARK-57086] Support `recoverPartitions` in `Catalog`
### What changes were proposed in this pull request?
This PR aims to support `recoverPartitions` in `Catalog`.
### Why are the changes needed?
For feature parity with `spark.catalog.recoverPartitions`.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs with a new test case.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.7
Closes #394 from dongjoon-hyun/SPARK-57086.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
Sources/SparkConnect/Catalog.swift | 14 ++++++++++++++
Tests/SparkConnectTests/CatalogTests.swift | 17 +++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/Sources/SparkConnect/Catalog.swift
b/Sources/SparkConnect/Catalog.swift
index ef1ee36..64d5574 100644
--- a/Sources/SparkConnect/Catalog.swift
+++ b/Sources/SparkConnect/Catalog.swift
@@ -633,6 +633,20 @@ public actor Catalog: Sendable {
try await df.count()
}
+ /// Recovers all the partitions in the directory of a table and update the
catalog.
+ /// - Parameter tableName: A qualified or unqualified name that designates a
table.
+ /// If no database identifier is provided, it refers to a table in the
current database.
+ public func recoverPartitions(_ tableName: String) async throws {
+ let df = getDataFrame({
+ var recoverPartitions = Spark_Connect_RecoverPartitions()
+ recoverPartitions.tableName = tableName
+ var catalog = Spark_Connect_Catalog()
+ catalog.catType = .recoverPartitions(recoverPartitions)
+ return catalog
+ })
+ try await df.count()
+ }
+
/// Analyzes the given table to compute statistics that can be used by the
query optimizer.
/// - Parameters:
/// - tableName: A qualified or unqualified name that designates a table.
diff --git a/Tests/SparkConnectTests/CatalogTests.swift
b/Tests/SparkConnectTests/CatalogTests.swift
index b6d6fb6..94c21dc 100644
--- a/Tests/SparkConnectTests/CatalogTests.swift
+++ b/Tests/SparkConnectTests/CatalogTests.swift
@@ -626,6 +626,23 @@ struct CatalogTests {
await spark.stop()
}
+ @Test
+ func recoverPartitions() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-",
with: "")
+ try await SQLHelper.withTable(spark, tableName)({
+ try await spark.sql(
+ "CREATE TABLE \(tableName) (id INT) USING PARQUET PARTITIONED BY (p
INT)"
+ ).count()
+ try await spark.catalog.recoverPartitions(tableName)
+ })
+
+ try await #require(throws: SparkConnectError.TableOrViewNotFound) {
+ try await spark.catalog.recoverPartitions("not_exist_table")
+ }
+ await spark.stop()
+ }
+
@Test
func analyzeTable() async throws {
let spark = try await SparkSession.builder.getOrCreate()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
