This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a008390469 [spark] Show format table partition statistics (#9501)
a008390469 is described below
commit a008390469ebb1721e18429540a78d457d7c9dbe
Author: Dapeng Sun(孙大鹏) <[email protected]>
AuthorDate: Tue Sep 1 10:20:04 2026 +0800
[spark] Show format table partition statistics (#9501)
---
.../paimon/spark/PaimonPartitionManagement.scala | 24 +++++++++++
.../sql/CatalogManagedPartitionAnalyzeTest.scala | 49 ++++++++++++++++++++++
2 files changed, 73 insertions(+)
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
index 392b127ff1..a02477ce6d 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
@@ -165,6 +165,30 @@ trait PaimonPartitionManagement extends
SupportsAtomicPartitionManagement with L
} else {
Map.empty[String, String].asJava
}
+ case formatTable: FormatTable =>
+ val partitionManager = formatTable.partitionManager()
+ if (partitionManager == null) {
+ Map.empty[String, String].asJava
+ } else {
+ val partitionSpec =
+ toPaimonPartition(ident, formatTable.partitionKeys().asScala.toSeq)
+ val partitions =
partitionManager.listPartitionsByNames(Seq(partitionSpec).asJava)
+ if (!partitions.isEmpty) {
+ val partition = partitions.get(0)
+ Map(
+ PartitionStatistics.FIELD_RECORD_COUNT ->
partition.recordCount().toString,
+ PartitionStatistics.FIELD_FILE_SIZE_IN_BYTES -> partition
+ .fileSizeInBytes()
+ .toString,
+ PartitionStatistics.FIELD_FILE_COUNT ->
partition.fileCount().toString,
+ PartitionStatistics.FIELD_LAST_FILE_CREATION_TIME -> partition
+ .lastFileCreationTime()
+ .toString
+ ).asJava
+ } else {
+ Map.empty[String, String].asJava
+ }
+ }
case _ =>
Map.empty[String, String].asJava
}
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionAnalyzeTest.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionAnalyzeTest.scala
index 3b2a2d1b9a..e915b6fedc 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionAnalyzeTest.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionAnalyzeTest.scala
@@ -338,6 +338,55 @@ class CatalogManagedPartitionAnalyzeTest extends
PaimonSparkTestWithRestCatalogB
}
}
+ test("SHOW TABLE EXTENDED displays catalog-managed Format Table partition
statistics") {
+ val tableName = "analyze_show_partition_statistics"
+ withTable(tableName) {
+ sql(s"""CREATE TABLE $tableName (id INT, payload STRING, dt STRING, hour
STRING)
+ |USING PARQUET
+ |PARTITIONED BY (dt, hour)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'metastore.partitioned-table' = 'true')
+ |""".stripMargin)
+ sql(s"""INSERT INTO ${qualified(tableName)} VALUES
+ |(1, 'a', '20260101', '00'), (2, 'b', '20260101', '00')
+ |""".stripMargin)
+ sql(
+ s"ANALYZE TABLE ${qualified(tableName)} " +
+ s"PARTITION (dt = '20260101', hour = '00') COMPUTE
STATISTICS").collect()
+
+ val information =
+ sql(
+ s"SHOW TABLE EXTENDED IN paimon.$dbName0 LIKE '$tableName' " +
+ s"PARTITION (dt = '20260101', hour = '00')")
+ .select("information")
+ .collect()
+ .head
+ .getString(0)
+
+
assert(information.contains(s"${PartitionStatistics.FIELD_RECORD_COUNT}=2"),
information)
+ val statistics = statisticsOf(tableName, "20260101", "00")
+ assert(statistics.fileCount() > 0L, statistics.toString)
+ assert(statistics.fileSizeInBytes() > 0L, statistics.toString)
+ assert(statistics.lastFileCreationTime() > 0L, statistics.toString)
+ assert(
+
information.contains(s"${PartitionStatistics.FIELD_FILE_COUNT}=${statistics.fileCount()}"),
+ information)
+ assert(
+ information.contains(
+
s"${PartitionStatistics.FIELD_FILE_SIZE_IN_BYTES}=${statistics.fileSizeInBytes()}"),
+ information)
+ assert(
+ information.contains(
+ s"${PartitionStatistics.FIELD_LAST_FILE_CREATION_TIME}=" +
+ s"${statistics.lastFileCreationTime()}"),
+ information)
+ assert(
+ information.matches("(?s).*Partition Statistics: 2 rows, [1-9]\\d*
bytes.*"),
+ information)
+ }
+ }
+
test("a full ANALYZE clamps non-positive statistics parallelism") {
Seq("zero" -> 0, "negative" -> -1).foreach {
case (label, parallelism) =>
