This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 5f76cab [SPARK-31980][SQL] Function sequence() fails if start and end
of range are equal dates
5f76cab is described below
commit 5f76cab1625f4365fe703514bdb09b2970f5a74a
Author: TJX2014 <[email protected]>
AuthorDate: Fri Jun 19 19:24:34 2020 -0700
[SPARK-31980][SQL] Function sequence() fails if start and end of range are
equal dates
### What changes were proposed in this pull request?
1. Add judge equal as bigger condition in
`org.apache.spark.sql.catalyst.expressions.Sequence.TemporalSequenceImpl#eval`
2. Unit test for interval `day`, `month`, `year`
### Why are the changes needed?
Bug exists when sequence input get same equal start and end dates, which
will occur `while loop` forever
### Does this PR introduce _any_ user-facing change?
Yes,
Before this PR, people will get a
`java.lang.ArrayIndexOutOfBoundsException`, when eval as below:
`sql("select sequence(cast('2011-03-01' as date), cast('2011-03-01' as
date), interval 1 year)").show(false)
`
### How was this patch tested?
Unit test.
Closes #28819 from TJX2014/master-SPARK-31980.
Authored-by: TJX2014 <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 177a380bcf1f56982760442e8418d28415bef8b0)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../catalyst/expressions/collectionOperations.scala | 4 ++--
.../expressions/CollectionExpressionsSuite.scala | 18 ++++++++++++++++++
2 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 4fd68dc..7a15bcc 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -2624,7 +2624,7 @@ object Sequence {
val maxEstimatedArrayLength =
getSequenceLength(startMicros, stopMicros, intervalStepInMicros)
- val stepSign = if (stopMicros > startMicros) +1 else -1
+ val stepSign = if (stopMicros >= startMicros) +1 else -1
val exclusiveItem = stopMicros + stepSign
val arr = new Array[T](maxEstimatedArrayLength)
var t = startMicros
@@ -2686,7 +2686,7 @@ object Sequence {
|
| $sequenceLengthCode
|
- | final int $stepSign = $stopMicros > $startMicros ? +1 : -1;
+ | final int $stepSign = $stopMicros >= $startMicros ? +1 : -1;
| final long $exclusiveItem = $stopMicros + $stepSign;
|
| $arr = new $elemType[$arrLength];
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 9177382..c8c97d3 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -1836,4 +1836,22 @@ class CollectionExpressionsSuite extends SparkFunSuite
with ExpressionEvalHelper
checkEvaluation(ArrayIntersect(empty, oneNull), Seq.empty)
checkEvaluation(ArrayIntersect(oneNull, empty), Seq.empty)
}
+
+ test("SPARK-31980: Start and end equal in month range") {
+ checkEvaluation(new Sequence(
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(stringToInterval("interval 1 day"))),
+ Seq(Date.valueOf("2018-01-01")))
+ checkEvaluation(new Sequence(
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(stringToInterval("interval 1 month"))),
+ Seq(Date.valueOf("2018-01-01")))
+ checkEvaluation(new Sequence(
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(stringToInterval("interval 1 year"))),
+ Seq(Date.valueOf("2018-01-01")))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
