Repository: spark Updated Branches: refs/heads/branch-2.4 56f706867 -> d05596ebd
[SPARK-25431][SQL][EXAMPLES] Fix function examples and the example results. ## What changes were proposed in this pull request? There are some mistakes in examples of newly added functions. Also the format of the example results are not unified. We should fix them. ## How was this patch tested? Manually executed the examples. Closes #22437 from ueshin/issues/SPARK-25431/fix_examples_2. Authored-by: Takuya UESHIN <[email protected]> Signed-off-by: hyukjinkwon <[email protected]> (cherry picked from commit 8cf6fd1c2342949916fedb5a7f712177b22585fa) Signed-off-by: hyukjinkwon <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d05596eb Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d05596eb Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d05596eb Branch: refs/heads/branch-2.4 Commit: d05596ebd57b24f8caf6d76a5c9a8684f45a41b6 Parents: 56f7068 Author: Takuya UESHIN <[email protected]> Authored: Mon Sep 17 20:40:42 2018 +0800 Committer: hyukjinkwon <[email protected]> Committed: Mon Sep 17 20:40:56 2018 +0800 ---------------------------------------------------------------------- .../expressions/collectionOperations.scala | 37 ++++++++++---------- .../expressions/complexTypeCreator.scala | 2 +- .../expressions/higherOrderFunctions.scala | 32 ++++++++--------- 3 files changed, 36 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/d05596eb/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index ea6fccc..cc9edcf 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -159,9 +159,9 @@ case class MapKeys(child: Expression) examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(2, 3, 4)); - [[1, 2], [2, 3], [3, 4]] + [{"0":1,"1":2},{"0":2,"1":3},{"0":3,"1":4}] > SELECT _FUNC_(array(1, 2), array(2, 3), array(3, 4)); - [[1, 2, 3], [2, 3, 4]] + [{"0":1,"1":2,"2":3},{"0":2,"1":3,"2":4}] """, since = "2.4.0") case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsInputTypes { @@ -348,7 +348,7 @@ case class MapValues(child: Expression) examples = """ Examples: > SELECT _FUNC_(map(1, 'a', 2, 'b')); - [(1,"a"),(2,"b")] + [{"key":1,"value":"a"},{"key":2,"value":"b"}] """, since = "2.4.0") case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInputTypes { @@ -516,7 +516,7 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp examples = """ Examples: > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(2, 'c', 3, 'd')); - [[1 -> "a"], [2 -> "b"], [2 -> "c"], [3 -> "d"]] + {1:"a",2:"c",3:"d"} """, since = "2.4.0") case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpression { @@ -1171,9 +1171,9 @@ case class ArraySort(child: Expression) extends UnaryExpression with ArraySortLi examples = """ Examples: > SELECT _FUNC_(array(1, 20, 3, 5)); - [3, 1, 5, 20] + [3,1,5,20] > SELECT _FUNC_(array(1, 20, null, 3)); - [20, null, 3, 1] + [20,null,3,1] """, note = "The function is non-deterministic.", since = "2.4.0") @@ -1256,7 +1256,7 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None) > SELECT _FUNC_('Spark SQL'); LQS krapS > SELECT _FUNC_(array(2, 1, 4, 3)); - [3, 4, 1, 2] + [3,4,1,2] """, since = "1.5.0", note = "Reverse logic for arrays is available since 2.4.0." @@ -2123,7 +2123,7 @@ case class ArrayPosition(left: Expression, right: Expression) > SELECT _FUNC_(array(1, 2, 3), 2); 2 > SELECT _FUNC_(map(1, 'a', 2, 'b'), 2); - "b" + b """, since = "2.4.0") case class ElementAt(left: Expression, right: Expression) extends GetMapValueUtil { @@ -2238,8 +2238,9 @@ case class ElementAt(left: Expression, right: Expression) extends GetMapValueUti > SELECT _FUNC_('Spark', 'SQL'); SparkSQL > SELECT _FUNC_(array(1, 2, 3), array(4, 5), array(6)); - | [1,2,3,4,5,6] - """) + [1,2,3,4,5,6] + """, + note = "Concat logic for arrays is available since 2.4.0.") case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpression { private def allowedTypes: Seq[AbstractDataType] = Seq(StringType, BinaryType, ArrayType) @@ -2427,7 +2428,7 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio usage = "_FUNC_(arrayOfArrays) - Transforms an array of arrays into a single array.", examples = """ Examples: - > SELECT _FUNC_(array(array(1, 2), array(3, 4)); + > SELECT _FUNC_(array(array(1, 2), array(3, 4))); [1,2,3,4] """, since = "2.4.0") @@ -2556,11 +2557,11 @@ case class Flatten(child: Expression) extends UnaryExpression { examples = """ Examples: > SELECT _FUNC_(1, 5); - [1, 2, 3, 4, 5] + [1,2,3,4,5] > SELECT _FUNC_(5, 1); - [5, 4, 3, 2, 1] + [5,4,3,2,1] > SELECT _FUNC_(to_date('2018-01-01'), to_date('2018-03-01'), interval 1 month); - [2018-01-01, 2018-02-01, 2018-03-01] + [2018-01-01,2018-02-01,2018-03-01] """, since = "2.4.0" ) @@ -2934,7 +2935,7 @@ object Sequence { examples = """ Examples: > SELECT _FUNC_('123', 2); - ['123', '123'] + ["123","123"] """, since = "2.4.0") case class ArrayRepeat(left: Expression, right: Expression) @@ -3421,7 +3422,7 @@ object ArrayBinaryLike { examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); - array(1, 2, 3, 5) + [1,2,3,5] """, since = "2.4.0") case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLike @@ -3632,7 +3633,7 @@ object ArrayUnion { examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); - array(1, 3) + [1,3] """, since = "2.4.0") case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBinaryLike @@ -3873,7 +3874,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); - array(2) + [2] """, since = "2.4.0") case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryLike http://git-wip-us.apache.org/repos/asf/spark/blob/d05596eb/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index aba9c6c..fd8b5e9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -248,7 +248,7 @@ case class CreateMap(children: Seq[Expression]) extends Expression { in keys should not be null""", examples = """ Examples: - > SELECT _FUNC_([1.0, 3.0], ['2', '4']); + > SELECT _FUNC_(array(1.0, 3.0), array('2', '4')); {1.0:"2",3.0:"4"} """, since = "2.4.0") case class MapFromArrays(left: Expression, right: Expression) http://git-wip-us.apache.org/repos/asf/spark/blob/d05596eb/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala index 2bb6b20..b07d946 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala @@ -209,9 +209,9 @@ trait MapBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction { examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x + 1); - array(2, 3, 4) + [2,3,4] > SELECT _FUNC_(array(1, 2, 3), (x, i) -> x + i); - array(1, 3, 5) + [1,3,5] """, since = "2.4.0") case class ArrayTransform( @@ -268,7 +268,7 @@ usage = "_FUNC_(expr, func) - Filters entries in a map using the function.", examples = """ Examples: > SELECT _FUNC_(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v); - [1 -> 0, 3 -> -1] + {1:0,3:-1} """, since = "2.4.0") case class MapFilter( @@ -318,7 +318,7 @@ case class MapFilter( examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 1); - array(1, 3) + [1,3] """, since = "2.4.0") case class ArrayFilter( @@ -499,10 +499,10 @@ case class ArrayAggregate( usage = "_FUNC_(expr, func) - Transforms elements in a map using the function.", examples = """ Examples: - > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); - map(array(2, 3, 4), array(1, 2, 3)) - > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v); - map(array(2, 4, 6), array(1, 2, 3)) + > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); + {2:1,3:2,4:3} + > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v); + {2:1,4:2,6:3} """, since = "2.4.0") case class TransformKeys( @@ -549,10 +549,10 @@ case class TransformKeys( usage = "_FUNC_(expr, func) - Transforms values in the map using the function.", examples = """ Examples: - > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); - map(array(1, 2, 3), array(2, 3, 4)) - > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v); - map(array(1, 2, 3), array(2, 4, 6)) + > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); + {1:2,2:3,3:4} + > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v); + {1:2,2:4,3:6} """, since = "2.4.0") case class TransformValues( @@ -777,11 +777,11 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression) examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); - array(('a', 1), ('b', 2), ('c', 3)) - > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y)); - array(4, 6) + [{"y":"a","x":1},{"y":"b","x":2},{"y":"c","x":3}] + > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y); + [4,6] > SELECT _FUNC_(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x, y)); - array('ad', 'be', 'cf') + ["ad","be","cf"] """, since = "2.4.0") // scalastyle:on line.size.limit --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
