This is an automated email from the ASF dual-hosted git repository.
yumwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e6c01cedbec [SPARK-42063][CORE] Register `byte[][]` to KyroSerializer
e6c01cedbec is described below
commit e6c01cedbecebdc40863e552862beee1bd959710
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Jan 14 21:45:43 2023 +0800
[SPARK-42063][CORE] Register `byte[][]` to KyroSerializer
### What changes were proposed in this pull request?
This PR aims to register `byte[][]` to `KryoSerializer`.
### Why are the changes needed?
Spark uses `byte[][]` internally and it causes failure when we use
`spark.kryo.registrationRequired=true`.
```
java.lang.IllegalArgumentException: Class is not registered: byte[][]
Note: To register this class use: kryo.register(byte[][].class);
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs with the newly added test case.
Closes #39569 from dongjoon-hyun/SPARK-42063.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Yuming Wang <[email protected]>
---
core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala | 1 +
.../test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala | 2 ++
2 files changed, 3 insertions(+)
diff --git
a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index ce45762d20e..0f40cd809d2 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -220,6 +220,7 @@ class KryoSerializer(conf: SparkConf)
kryo.register(Utils.classForName("scala.collection.immutable.$colon$colon"))
kryo.register(Utils.classForName("scala.collection.immutable.Map$EmptyMap$"))
kryo.register(classOf[ArrayBuffer[Any]])
+ kryo.register(classOf[Array[Array[Byte]]])
// We can't load those class directly in order to avoid unnecessary jar
dependencies.
// We load them safely, ignore it if the class not found.
diff --git
a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index dd2340a8672..8c4ba583070 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -104,6 +104,7 @@ class KryoSerializerSuite extends SparkFunSuite with
SharedSparkContext {
check(java.lang.Long.MAX_VALUE)
check(java.lang.Long.MIN_VALUE)
check[String](null)
+ check(Array(1.toByte))
check(Array(1, 2, 3))
check(Array(1L, 2L, 3L))
check(Array(1.0, 2.0, 3.0))
@@ -114,6 +115,7 @@ class KryoSerializerSuite extends SparkFunSuite with
SharedSparkContext {
check(Array('a', 'b', 'c'))
check(Array.empty[Int])
check(Array(Array("1", "2"), Array("1", "2", "3", "4")))
+ check(Array(Array(1.toByte)))
}
test("pairs") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
