Repository: spark Updated Branches: refs/heads/master 8a9ca1924 -> 6e2701815
[SPARK-18260] Make from_json null safe ## What changes were proposed in this pull request? `from_json` is currently not safe against `null` rows. This PR adds a fix and a regression test for it. ## How was this patch tested? Regression test Author: Burak Yavuz <[email protected]> Closes #15771 from brkyvz/json_fix. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6e270181 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6e270181 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6e270181 Branch: refs/heads/master Commit: 6e2701815761d5870111cb56300e30d3059b39ed Parents: 8a9ca19 Author: Burak Yavuz <[email protected]> Authored: Sat Nov 5 00:07:51 2016 -0700 Committer: Reynold Xin <[email protected]> Committed: Sat Nov 5 00:07:51 2016 -0700 ---------------------------------------------------------------------- .../spark/sql/catalyst/expressions/jsonExpressions.scala | 4 +++- .../sql/catalyst/expressions/JsonExpressionsSuite.scala | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/6e270181/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index e034735..89fe7c4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -498,7 +498,9 @@ case class JsonToStruct(schema: StructType, options: Map[String, String], child: override def children: Seq[Expression] = child :: Nil override def eval(input: InternalRow): Any = { - try parser.parse(child.eval(input).toString).head catch { + val json = child.eval(input) + if (json == null) return null + try parser.parse(json.toString).head catch { case _: SparkSQLJsonProcessingException => null } } http://git-wip-us.apache.org/repos/asf/spark/blob/6e270181/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index f9db649..3bfa0bf 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -344,6 +344,14 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { ) } + test("from_json null input column") { + val schema = StructType(StructField("a", IntegerType) :: Nil) + checkEvaluation( + JsonToStruct(schema, Map.empty, Literal(null)), + null + ) + } + test("to_json") { val schema = StructType(StructField("a", IntegerType) :: Nil) val struct = Literal.create(create_row(1), schema) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
