Repository: spark
Updated Branches:
refs/heads/branch-2.0 a93f04dd4 -> 69f39552d
[SPARK-15462][SQL][TEST] unresolved === false` is enough in testcases.
## What changes were proposed in this pull request?
In only `catalyst` module, there exists 8 evaluation test cases on unresolved
expressions. But, in real-world situation, those cases doesn't happen since
they occurs exceptions before evaluations.
```scala
scala> sql("select format_number(null, 3)")
res0: org.apache.spark.sql.DataFrame = [format_number(CAST(NULL AS DOUBLE), 3):
string]
scala> sql("select format_number(cast(null as NULL), 3)")
org.apache.spark.sql.catalyst.parser.ParseException:
DataType null() is not supported.(line 1, pos 34)
```
This PR makes those testcases more realistic.
```scala
- checkEvaluation(FormatNumber(Literal.create(null, NullType), Literal(3)),
null)
+ assert(FormatNumber(Literal.create(null, NullType), Literal(3)).resolved
=== false)
```
Also, this PR also removes redundant `resolved` checking in
`FoldablePropagation` optimizer.
## How was this patch tested?
Pass the modified Jenkins tests.
Author: Dongjoon Hyun <[email protected]>
Closes #13241 from dongjoon-hyun/SPARK-15462.
(cherry picked from commit f39621c998a0fe91a5115f3f843c3ca8dd71c1ab)
Signed-off-by: Wenchen Fan <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/69f39552
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/69f39552
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/69f39552
Branch: refs/heads/branch-2.0
Commit: 69f39552d66eb3fe1cd096d6f8380f3f66de91e8
Parents: a93f04d
Author: Dongjoon Hyun <[email protected]>
Authored: Sat May 21 08:11:14 2016 -0700
Committer: Wenchen Fan <[email protected]>
Committed: Sat May 21 08:11:24 2016 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 2 +-
.../org/apache/spark/sql/catalyst/expressions/CastSuite.scala | 7 -------
.../spark/sql/catalyst/expressions/DateExpressionsSuite.scala | 6 +++---
.../sql/catalyst/expressions/StringExpressionsSuite.scala | 2 +-
4 files changed, 5 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/69f39552/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index a6fb34c..5e998d6 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -673,7 +673,7 @@ object FoldablePropagation extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = {
val foldableMap = AttributeMap(plan.flatMap {
case Project(projectList, _) => projectList.collect {
- case a: Alias if a.resolved && a.child.foldable => (a.toAttribute, a)
+ case a: Alias if a.child.foldable => (a.toAttribute, a)
}
case _ => Nil
})
http://git-wip-us.apache.org/repos/asf/spark/blob/69f39552/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index 43af359..dfda7c5 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -548,7 +548,6 @@ class CastSuite extends SparkFunSuite with
ExpressionEvalHelper {
{
val ret = cast(array_notNull, ArrayType(BooleanType, containsNull =
false))
assert(ret.resolved === false)
- checkEvaluation(ret, Seq(null, true, false))
}
{
@@ -607,7 +606,6 @@ class CastSuite extends SparkFunSuite with
ExpressionEvalHelper {
{
val ret = cast(map_notNull, MapType(StringType, BooleanType,
valueContainsNull = false))
assert(ret.resolved === false)
- checkEvaluation(ret, Map("a" -> null, "b" -> true, "c" -> false))
}
{
val ret = cast(map_notNull, MapType(IntegerType, StringType,
valueContainsNull = true))
@@ -714,7 +712,6 @@ class CastSuite extends SparkFunSuite with
ExpressionEvalHelper {
StructField("b", BooleanType, nullable = true),
StructField("c", BooleanType, nullable = false))))
assert(ret.resolved === false)
- checkEvaluation(ret, InternalRow(null, true, false))
}
{
@@ -755,10 +752,6 @@ class CastSuite extends SparkFunSuite with
ExpressionEvalHelper {
StructField("l", LongType, nullable = true)))))))
assert(ret.resolved === false)
- checkEvaluation(ret, Row(
- Seq(123, null, null),
- Map("a" -> null, "b" -> true, "c" -> false),
- Row(0L)))
}
test("cast between string and interval") {
http://git-wip-us.apache.org/repos/asf/spark/blob/69f39552/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 53c66d8..6118a34 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -143,7 +143,7 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
test("Seconds") {
- checkEvaluation(Second(Literal.create(null, DateType)), null)
+ assert(Second(Literal.create(null, DateType)).resolved === false)
checkEvaluation(Second(Cast(Literal(d), TimestampType)), 0)
checkEvaluation(Second(Cast(Literal(sdf.format(d)), TimestampType)), 15)
checkEvaluation(Second(Literal(ts)), 15)
@@ -176,7 +176,7 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
test("Hour") {
- checkEvaluation(Hour(Literal.create(null, DateType)), null)
+ assert(Hour(Literal.create(null, DateType)).resolved === false)
checkEvaluation(Hour(Cast(Literal(d), TimestampType)), 0)
checkEvaluation(Hour(Cast(Literal(sdf.format(d)), TimestampType)), 13)
checkEvaluation(Hour(Literal(ts)), 13)
@@ -195,7 +195,7 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
test("Minute") {
- checkEvaluation(Minute(Literal.create(null, DateType)), null)
+ assert(Minute(Literal.create(null, DateType)).resolved === false)
checkEvaluation(Minute(Cast(Literal(d), TimestampType)), 0)
checkEvaluation(Minute(Cast(Literal(sdf.format(d)), TimestampType)), 10)
checkEvaluation(Minute(Literal(ts)), 10)
http://git-wip-us.apache.org/repos/asf/spark/blob/69f39552/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index 2cf8ca7..c09c64f 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -688,7 +688,7 @@ class StringExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
Literal(Decimal(123123324123L) * Decimal(123123.21234d)), Literal(4)),
"15,159,339,180,002,773.2778")
checkEvaluation(FormatNumber(Literal.create(null, IntegerType),
Literal(3)), null)
- checkEvaluation(FormatNumber(Literal.create(null, NullType), Literal(3)),
null)
+ assert(FormatNumber(Literal.create(null, NullType), Literal(3)).resolved
=== false)
}
test("find in set") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
