This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new e12e9b2e3431 [SPARK-57831][SQL] Align Hive-metastore compatibility
classification for nanosecond-precision timestamp types
e12e9b2e3431 is described below
commit e12e9b2e343146705191c110a6e9cf9540fef759
Author: Stevo Mitric <[email protected]>
AuthorDate: Thu Jul 9 09:09:33 2026 +0800
[SPARK-57831][SQL] Align Hive-metastore compatibility classification for
nanosecond-precision timestamp types
### What changes were proposed in this pull request?
Classify the nanosecond-precision timestamp types as Hive-incompatible in
`HiveExternalCatalog.isHiveCompatibleDataType`.
### Why are the changes needed?
These are Spark-specific types with no Hive metastore equivalent.
Previously they fell through to the `case _ => true` default, so a table using
them could take the Hive-compatible metastore path and store a non-standard
`timestamp_ntz(9)` / `timestamp_ltz(9)` type string in the HMS `FieldSchema`.
### Does this PR introduce _any_ user-facing change?
No. This affects only how nanosecond-typed tables are persisted to the Hive
metastore.
### How was this patch tested?
Added coverage in `MetastoreDataSourcesSuite` exercising create/reload of
tables with nanosecond timestamp columns.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.8
Closes #57084 from stevomitric/stevomitric/spark-57831-nanos.
Authored-by: Stevo Mitric <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 10d5950df194bbfc8356392c7cfbc91a88087b77)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../spark/sql/hive/HiveExternalCatalog.scala | 1 +
.../spark/sql/hive/MetastoreDataSourcesSuite.scala | 97 ++++++++++++++++++++++
2 files changed, 98 insertions(+)
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 5d3a872d047c..9a00cc5516a7 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -1541,6 +1541,7 @@ object HiveExternalCatalog {
private[spark] def isHiveCompatibleDataType(dt: DataType): Boolean = dt
match {
case _: AnsiIntervalType => false
case _: TimestampNTZType => false
+ case _: AnyTimestampNanoType => false
case _: VariantType => false
case s: StructType =>
s.forall { f =>
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index 2c79f5f03702..405a14fc2afa 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.errors.QueryErrorsBase
import org.apache.spark.sql.execution.command.CreateTableCommand
import org.apache.spark.sql.execution.datasources.{HadoopFsRelation,
LogicalRelationWithTable}
import org.apache.spark.sql.hive.HiveExternalCatalog._
+import org.apache.spark.sql.hive.client.HiveClientImpl
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.StaticSQLConf._
@@ -1573,6 +1574,102 @@ class MetastoreDataSourcesSuite extends QueryTest
}
}
+ test("SPARK-57831: nanosecond timestamp columns are stored in Spark-specific
format " +
+ "(not Hive compatible)") {
+ withSQLConf(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
+ withTable("t") {
+ val logAppender = new LogAppender(
+ "Check that nanosecond timestamp types are reported as Hive
incompatible")
+ logAppender.setThreshold(Level.WARN)
+ withLogAppender(logAppender) {
+ sql(
+ """
+ |CREATE TABLE t(
+ | a TIMESTAMP_NTZ(9),
+ | b TIMESTAMP_LTZ(9),
+ | c TIMESTAMP_NTZ(7),
+ | d TIMESTAMP_LTZ(8)
+ |) USING Parquet""".stripMargin)
+ }
+ // (a) The WARN message lists all four nanos types as Hive
incompatible. Their
+ // simpleString is the parameterized typeName (e.g.
"timestamp_ntz(9)").
+ val actualMessages = logAppender.loggingEvents
+ .map(_.getMessage.getFormattedMessage)
+ .filter(_.contains("incompatible"))
+ assert(actualMessages.exists(m =>
+ m.contains("timestamp_ntz(9)") && m.contains("timestamp_ltz(9)") &&
+ m.contains("timestamp_ntz(7)") && m.contains("timestamp_ltz(8)")))
+ // (b) Because the types are Hive incompatible, the HMS FieldSchema is
the dummy
+ // array<string> and the real schema is persisted in table properties.
+ assert(hiveClient.getTable("default", "t").schema
+ .forall(_.dataType == ArrayType(StringType)))
+ // (c) The reloaded logical schema round-trips back to the real nanos
types.
+ assert(sql("SELECT * FROM t").schema ===
+ StructType(Seq(
+ StructField("a", TimestampNTZNanosType(9)),
+ StructField("b", TimestampLTZNanosType(9)),
+ StructField("c", TimestampNTZNanosType(7)),
+ StructField("d", TimestampLTZNanosType(8)))))
+ }
+ }
+ }
+
+ test("SPARK-57831: nanosecond timestamp partition columns are stored in
Spark-specific " +
+ "format (not Hive compatible)") {
+ withSQLConf(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
+ Seq(
+ "TIMESTAMP_NTZ(9)" -> TimestampNTZNanosType(9),
+ "TIMESTAMP_LTZ(8)" -> TimestampLTZNanosType(8)
+ ).foreach { case (typeStr, dt) =>
+ withTable("t") {
+ val logAppender = new LogAppender(
+ "Check that nanosecond timestamp partition columns are Hive
incompatible")
+ logAppender.setThreshold(Level.WARN)
+ withLogAppender(logAppender) {
+ sql(
+ s"""
+ |CREATE TABLE t(id INT, p $typeStr)
+ |USING Parquet
+ |PARTITIONED BY (p)""".stripMargin)
+ }
+ // (a) The nanos partition column alone makes the table Hive
incompatible: the WARN
+ // lists its parameterized simpleString (e.g. "timestamp_ntz(9)").
This is driven by
+ // the partition column, since the data column `id` is itself Hive
compatible.
+ val actualMessages = logAppender.loggingEvents
+ .map(_.getMessage.getFormattedMessage)
+ .filter(_.contains("incompatible"))
+ assert(actualMessages.exists(_.contains(dt.simpleString)))
+ // (b) The raw HMS schema is the dummy array<string>: the data
column is the empty
+ // placeholder schema and the nanos partition column is stored via
the
+ // INCOMPATIBLE_PARTITION_TYPE_PLACEHOLDER (HiveClientImpl), which
the raw catalog view
+ // filters out. The real schema is restored from table properties in
(c).
+ assert(hiveClient.getTable("default", "t").schema
+ .forall(_.dataType == ArrayType(StringType)))
+ // (c) The reloaded logical schema round-trips back to the real
nanos partition type,
+ // with the partition column placed at the end of the schema.
+ assert(sql("SELECT * FROM t").schema ===
+ StructType(Seq(
+ StructField("id", IntegerType),
+ StructField("p", dt))))
+ }
+ }
+ }
+ }
+
+ // Note: this is parser/serde round-trip coverage for the nanos type
strings; it passes
+ // with or without the isHiveCompatibleDataType change and does not by
itself guard the
+ // regression. The load-bearing test is the "stored in Spark-specific
format" case above.
+ test("SPARK-57831: nanosecond timestamp FieldSchema round-trips via
to/fromHiveColumn") {
+ withSQLConf(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
+ for (p <- 7 to 9; dt <- Seq(TimestampNTZNanosType(p),
TimestampLTZNanosType(p))) {
+ val field = StructField("c", dt, nullable = true)
+ val hiveCol = HiveClientImpl.toHiveColumn(field)
+ val back = HiveClientImpl.fromHiveColumn(hiveCol)
+ assert(back.dataType === dt)
+ }
+ }
+ }
+
private def withDebugMode(f: => Unit): Unit = {
val previousValue = sparkSession.sparkContext.conf.get(DEBUG_MODE)
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
