This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 553a1906a131 [SPARK-57810][SQL] Infer nanosecond-precision timestamp
types in XML schema inference
553a1906a131 is described below
commit 553a1906a13144548e588af71be79186411ac452
Author: Anupam Yadav <[email protected]>
AuthorDate: Mon Jul 13 22:11:13 2026 +0800
[SPARK-57810][SQL] Infer nanosecond-precision timestamp types in XML schema
inference
### What changes were proposed in this pull request?
Extends XML schema inference (`XmlInferSchema`) to infer
nanosecond-precision timestamp types. A timestamp string with more than 6
fractional-second digits (with timezone info) now infers as the nanosecond LTZ
timestamp type, mirroring the existing nanosecond NTZ inference in the same
file and the JSON/CSV nanosecond-inference approach. Behavior is gated on the
existing nanosecond-timestamp config; values with <= 6 fractional digits still
infer as the microsecond `TimestampType`.
### Why are the changes needed?
Part of nanosecond-precision timestamp support (SPARK-56822). XML inference
previously truncated sub-microsecond timestamps to microsecond precision; it
should infer the nanosecond type when the data warrants it, consistent with
JSON and CSV.
### Does this PR introduce _any_ user-facing change?
Yes - with nanosecond timestamp types enabled, XML schema inference can
infer a nanosecond-precision timestamp type for sub-microsecond timestamps
(previously inferred as microsecond `TimestampType`).
### How was this patch tested?
`XmlInferSchemaSuite`: new tests covering pure-nanosecond inference,
pure-microsecond (unchanged), mixed widening, and string fallback.
### Was this patch authored or co-authored using generative AI tooling?
Authored with assistance by Claude Opus 4.8.
Closes #56935 from yadavay-amzn/SPARK-57810.
Authored-by: Anupam Yadav <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 00ba61d477abce9418d706b400ac27bc69c42714)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../spark/sql/catalyst/xml/XmlInferSchema.scala | 14 +++-
.../datasources/xml/XmlInferSchemaSuite.scala | 85 ++++++++++++++++++++++
2 files changed, 97 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
index b4c942326702..234de9160421 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
@@ -390,7 +390,8 @@ class XmlInferSchema(private val options: XmlOptions,
private val caseSensitive:
// would make the inferred type differ from the legacy path and depend
on row order.
// Re-entering at the top yields the same representative as from-scratch
inference (which
// reaches the temporal parsers through `tryParseDouble`), so the merged
type matches.
- case _: TimeType | DateType | TimestampNTZType | _:
TimestampNTZNanosType | TimestampType =>
+ case _: TimeType | DateType | TimestampNTZType | _:
TimestampNTZNanosType |
+ TimestampType | _: TimestampLTZNanosType =>
tryParseTime(value)
case StringType => StringType
case other: DataType =>
@@ -740,7 +741,16 @@ class XmlInferSchema(private val options: XmlOptions,
private val caseSensitive:
case _: IllegalArgumentException => false
}
if (isTimestamp) {
- TimestampType
+ // Prefer nanosecond type when there is a nonzero sub-microsecond
component
+ // (nanosWithinMicro != 0) that TimestampType cannot represent.
+ val hasSubMicro = SQLConf.get.timestampNanosTypesEnabled &&
+ timestampFormatter.parseNanosOptional(field, 9)
+ .exists(_.nanosWithinMicro != 0)
+ if (hasSubMicro) {
+ TimestampLTZNanosType(9)
+ } else {
+ TimestampType
+ }
} else {
tryParseBoolean(field)
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlInferSchemaSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlInferSchemaSuite.scala
index 2c72825167f0..18cfe17d433e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlInferSchemaSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlInferSchemaSuite.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.types.{
StringType,
StructField,
StructType,
+ TimestampLTZNanosType,
TimestampType,
TimeType
}
@@ -785,6 +786,90 @@ class XmlInferSchemaSuite
assert(incremental === batch, s"incremental and batch schemas differ
for: $xml")
}
}
+
+ test("SPARK-57810: XML infers nanosecond LTZ timestamps from sub-microsecond
fractional digits") {
+ withSQLConf(
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+ SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_LTZ") {
+ // A timestamp with >6 fractional digits and timezone info should infer
as
+ // TimestampLTZNanosType(9) when nanos types are enabled.
+ val xmlNanos = Seq(
+ """<ROW><ts>2025-06-15T12:30:45.123456789+00:00</ts></ROW>""")
+ val df = readData(xmlNanos)
+ assert(df.schema("ts").dataType === TimestampLTZNanosType(9),
+ s"Expected TimestampLTZNanosType(9), got ${df.schema("ts").dataType}")
+ }
+ }
+
+ test("SPARK-57810: XML infers TimestampType for <=6 fractional digits even
with nanos enabled") {
+ withSQLConf(
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+ SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_LTZ") {
+ // A timestamp with exactly 6 fractional digits (microsecond precision)
should still
+ // infer as TimestampType, not nanos type.
+ val xmlMicros = Seq(
+ """<ROW><ts>2025-06-15T12:30:45.123456+00:00</ts></ROW>""")
+ val df = readData(xmlMicros)
+ assert(df.schema("ts").dataType === TimestampType,
+ s"Expected TimestampType, got ${df.schema("ts").dataType}")
+ }
+ }
+
+ test("SPARK-57810: XML inferred type is TimestampType for mixed nano/micro
LTZ rows") {
+ // When some rows have >6 fractional digits (nano) and others have <=6
(micro), the inferred
+ // type must widen to TimestampType since compatibleType merges LTZ nanos
+ LTZ to LTZ.
+ withSQLConf(
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+ SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_LTZ") {
+ val xmlMixed = Seq(
+ """<ROW><ts>2025-06-15T12:30:45.123456789+00:00</ts></ROW>""",
+ """<ROW><ts>2025-06-15T12:30:45.123456+00:00</ts></ROW>""")
+ val df = readData(xmlMixed)
+ assert(df.schema("ts").dataType === TimestampType,
+ s"Expected TimestampType for mixed nano/micro, got
${df.schema("ts").dataType}")
+ }
+ }
+
+ test("SPARK-57810: LTZ nano timestamp + non-datetime field widens to
StringType") {
+ withSQLConf(
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+ SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_LTZ") {
+ val xmlNanoAndString = Seq(
+ """<ROW><ts>2025-06-15T12:30:45.123456789+00:00</ts></ROW>""",
+ """<ROW><ts>not-a-timestamp</ts></ROW>""")
+ val df = readData(xmlNanoAndString)
+ assert(df.schema("ts").dataType === StringType,
+ s"Expected StringType for nano + non-datetime, got
${df.schema("ts").dataType}")
+ }
+ }
+
+ test("SPARK-57810: nanosecond-precision timestamp infers as TimestampType
when config is off") {
+ // Production default: TIMESTAMP_NANOS_TYPES_ENABLED = false.
+ // A nano-precision value must still infer as plain TimestampType (micros).
+ withSQLConf(
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "false",
+ SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_LTZ") {
+ val xmlNanos = Seq(
+ """<ROW><ts>2020-01-01T12:00:00.123456789+00:00</ts></ROW>""")
+ val df = readData(xmlNanos)
+ assert(df.schema("ts").dataType === TimestampType,
+ s"Expected TimestampType with config off, got
${df.schema("ts").dataType}")
+ }
+ }
+
+ test("SPARK-57810: 7-digit trailing-zero fractional seconds infers as
TimestampType not nanos") {
+ // .1234560 has 7 fractional digits but nanosWithinMicro == 0.
+ // Must infer as TimestampType, not nanos -- semantics are 'nonzero
sub-microsecond value'.
+ withSQLConf(
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+ SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_LTZ") {
+ val xmlTrailingZero = Seq(
+ """<ROW><ts>2020-01-01T12:00:00.1234560+00:00</ts></ROW>""")
+ val df = readData(xmlTrailingZero)
+ assert(df.schema("ts").dataType === TimestampType,
+ s"Expected TimestampType for trailing-zero 7 digits, got
${df.schema("ts").dataType}")
+ }
+ }
}
trait XmlSchemaInferenceCaseSensitivityTests extends QueryTest {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
