Repository: spark Updated Branches: refs/heads/master 19ad18638 -> dce2f8c9d
[SPARK-8654][SQL] Analysis exception when using NULL IN (...) : invalid cast In the analysis phase , while processing the rules for IN predicate, we compare the in-list types to the lhs expression type and generate cast operation if necessary. In the case of NULL [NOT] IN expr1 , we end up generating cast between in list types to NULL like cast (1 as NULL) which is not a valid cast. The fix is to find a common type between LHS and RHS expressions and cast all the expression to the common type. Author: Dilip Biswal <[email protected]> This patch had conflicts when merged, resolved by Committer: Michael Armbrust <[email protected]> Closes #9036 from dilipbiswal/spark_8654_new. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/dce2f8c9 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/dce2f8c9 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/dce2f8c9 Branch: refs/heads/master Commit: dce2f8c9d74bda46f3c7a7ebe7372b04d6b17a61 Parents: 19ad186 Author: Dilip Biswal <[email protected]> Authored: Wed Oct 21 14:29:03 2015 -0700 Committer: Michael Armbrust <[email protected]> Committed: Wed Oct 21 14:29:03 2015 -0700 ---------------------------------------------------------------------- .../catalyst/analysis/HiveTypeCoercion.scala | 11 ++++++++-- .../sql/catalyst/analysis/AnalysisSuite.scala | 21 ++++++++++++++++++++ .../analysis/HiveTypeCoercionSuite.scala | 3 ++- 3 files changed, 32 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/dce2f8c9/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala index 87a3845..1140150 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala @@ -304,7 +304,11 @@ object HiveTypeCoercion { } /** - * Convert all expressions in in() list to the left operator type + * Convert the value and in list expressions to the common operator type + * by looking at all the argument types and finding the closest one that + * all the arguments can be cast to. When no common operator type is found + * the original expression will be returned and an Analysis Exception will + * be raised at type checking phase. */ object InConversion extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan resolveExpressions { @@ -312,7 +316,10 @@ object HiveTypeCoercion { case e if !e.childrenResolved => e case i @ In(a, b) if b.exists(_.dataType != a.dataType) => - i.makeCopy(Array(a, b.map(Cast(_, a.dataType)))) + findWiderCommonType(i.children.map(_.dataType)) match { + case Some(finalDataType) => i.withNewChildren(i.children.map(Cast(_, finalDataType))) + case None => i + } } } http://git-wip-us.apache.org/repos/asf/spark/blob/dce2f8c9/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index 0a1fa74..71d2939 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -153,4 +153,25 @@ class AnalysisSuite extends AnalysisTest { checkAnalysis(plan, expected) } + + test("SPARK-8654: invalid CAST in NULL IN(...) expression") { + val plan = Project(Alias(In(Literal(null), Seq(Literal(1), Literal(2))), "a")() :: Nil, + LocalRelation() + ) + assertAnalysisSuccess(plan) + } + + test("SPARK-8654: different types in inlist but can be converted to a commmon type") { + val plan = Project(Alias(In(Literal(null), Seq(Literal(1), Literal(1.2345))), "a")() :: Nil, + LocalRelation() + ) + assertAnalysisSuccess(plan) + } + + test("SPARK-8654: check type compatibility error") { + val plan = Project(Alias(In(Literal(null), Seq(Literal(true), Literal(1))), "a")() :: Nil, + LocalRelation() + ) + assertAnalysisError(plan, Seq("data type mismatch: Arguments must be same type")) + } } http://git-wip-us.apache.org/repos/asf/spark/blob/dce2f8c9/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala index 6f33ab7..d3fafaa 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala @@ -470,7 +470,8 @@ class HiveTypeCoercionSuite extends PlanTest { ) ruleTest(inConversion, In(Literal("a"), Seq(Literal(1), Literal("b"))), - In(Literal("a"), Seq(Cast(Literal(1), StringType), Cast(Literal("b"), StringType))) + In(Cast(Literal("a"), StringType), + Seq(Cast(Literal(1), StringType), Cast(Literal("b"), StringType))) ) } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
