This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 83db3984761b [SPARK-51881][SQL] Make AvroOptions comparable
83db3984761b is described below
commit 83db3984761bfed528bc0b98aeb65f68cd122448
Author: Vladimir Golubev <vladimir.golu...@databricks.com>
AuthorDate: Fri Apr 25 07:56:49 2025 +0900
[SPARK-51881][SQL] Make AvroOptions comparable
### What changes were proposed in this pull request?
Make AvroOptions comparable. Hadoop `Configuration` doesn't have equals,
but it's enough to compare `parameters`.
### Why are the changes needed?
To properly compare single-pass/fixed-point Analyzer logical plans.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #50683 from
vladimirg-db/vladimir-golubev_data/make-avro-options-comparable.
Authored-by: Vladimir Golubev <vladimir.golu...@databricks.com>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
.../org/apache/spark/sql/avro/AvroOptions.scala | 32 ++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
b/sql/core/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
index f63014d6ea9e..ab3607d1bd7a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.avro
import java.net.URI
+import java.util.HashMap
import org.apache.avro.Schema
import org.apache.hadoop.conf.Configuration
@@ -146,6 +147,37 @@ private[sql] class AvroOptions(
RECURSIVE_FIELD_MAX_DEPTH,
s"Should not be greater than $RECURSIVE_FIELD_MAX_DEPTH_LIMIT.")
}
+
+ /**
+ * [[hadoop.conf.Configuration]] is not comparable so we turn it into a map
for [[equals]] and
+ * [[hashCode]].
+ */
+ @transient private lazy val comparableConf = {
+ val iter = conf.iterator()
+ val result = new HashMap[String, String]
+ while (iter.hasNext()) {
+ val entry = iter.next()
+ result.put(entry.getKey(), entry.getValue())
+ }
+ result
+ }
+
+ override def equals(other: Any): Boolean = {
+ other match {
+ case that: AvroOptions =>
+ this.parameters == that.parameters &&
+ this.comparableConf == that.comparableConf
+ case _ => false
+ }
+ }
+
+ override def hashCode(): Int = {
+ val prime = 31
+ var result = 1
+ result = prime * result + parameters.hashCode
+ result = prime * result + comparableConf.hashCode
+ result
+ }
}
private[sql] object AvroOptions extends DataSourceOptions {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
