This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8bfaa62 [SPARK-31175][SQL] Avoid creating reverse comparator for each
compare in InterpretedOrdering
8bfaa62 is described below
commit 8bfaa62f2fcc942dd99a63b20366167277bce2a1
Author: yi.wu <[email protected]>
AuthorDate: Wed Mar 18 23:56:48 2020 +0800
[SPARK-31175][SQL] Avoid creating reverse comparator for each compare in
InterpretedOrdering
### What changes were proposed in this pull request?
Prpend `-` to the compare result instead of creating a new reverse
comparator for each compare when sorting in DESC order in InterpretedOrdering.
### Why are the changes needed?
Currently, we'll create a new reverse comparator for each compare in
InterpretedOrdering, which could generate lots of small and instant object and
hurt JVM when there're plenty of data.
### Does this PR introduce any user-facing change?
No.
### How was this patch tested?
Pass Jenkins.
Closes #27938 from Ngone51/reverse_comparator.
Authored-by: yi.wu <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../scala/org/apache/spark/sql/catalyst/expressions/ordering.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ordering.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ordering.scala
index fa2978c..f387e17 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ordering.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ordering.scala
@@ -59,15 +59,15 @@ class InterpretedOrdering(ordering: Seq[SortOrder]) extends
BaseOrdering {
case dt: AtomicType if order.direction == Ascending =>
dt.ordering.asInstanceOf[Ordering[Any]].compare(left, right)
case dt: AtomicType if order.direction == Descending =>
- dt.ordering.asInstanceOf[Ordering[Any]].reverse.compare(left,
right)
+ - dt.ordering.asInstanceOf[Ordering[Any]].compare(left, right)
case a: ArrayType if order.direction == Ascending =>
a.interpretedOrdering.asInstanceOf[Ordering[Any]].compare(left,
right)
case a: ArrayType if order.direction == Descending =>
-
a.interpretedOrdering.asInstanceOf[Ordering[Any]].reverse.compare(left, right)
+ - a.interpretedOrdering.asInstanceOf[Ordering[Any]].compare(left,
right)
case s: StructType if order.direction == Ascending =>
s.interpretedOrdering.asInstanceOf[Ordering[Any]].compare(left,
right)
case s: StructType if order.direction == Descending =>
-
s.interpretedOrdering.asInstanceOf[Ordering[Any]].reverse.compare(left, right)
+ - s.interpretedOrdering.asInstanceOf[Ordering[Any]].compare(left,
right)
case other =>
throw new IllegalArgumentException(s"Type $other does not support
ordered operations")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
