Repository: zeppelin Updated Branches: refs/heads/master ba4418d00 -> 82d836fd5
[ZEPPELIN-1801] To force end table - append %text ### What is this PR for? It seems there is a bug in scio introduced by recent changes. Table rendered from scio helper methods would include interpreter results as well as the data. ### What type of PR is it? Bug Fix ### Todos ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-1801 ### How should this be tested? Try: ``` %beam.scio val (sc, args) = ContextAndArgs(argz) sc.parallelize(Seq("foo", "foo", "bar")).countByValue.closeAndDisplay() ``` before and after the change - and notice extra meaningless columns before the fix. ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Rafal Wojdyla <r...@spotify.com> Closes #1755 from ravwojdyla/fix_1801 and squashes the following commits: 00ffdea [Rafal Wojdyla] To force end table - append %text in scio helpers Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/82d836fd Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/82d836fd Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/82d836fd Branch: refs/heads/master Commit: 82d836fd5a2c045f8f79d65f78396bf9c4818e85 Parents: ba4418d Author: Rafal Wojdyla <r...@spotify.com> Authored: Tue Dec 13 11:58:56 2016 -0500 Committer: Felix Cheung <felixche...@apache.org> Committed: Mon Dec 19 19:38:07 2016 -0800 ---------------------------------------------------------------------- .../apache/zeppelin/scio/DisplayHelpers.scala | 7 ++ .../zeppelin/scio/DisplayHelpersTest.scala | 71 ++++++++++++++------ 2 files changed, 59 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/82d836fd/scio/src/main/scala/org/apache/zeppelin/scio/DisplayHelpers.scala ---------------------------------------------------------------------- diff --git a/scio/src/main/scala/org/apache/zeppelin/scio/DisplayHelpers.scala b/scio/src/main/scala/org/apache/zeppelin/scio/DisplayHelpers.scala index cdf718c..8dee3ab 100644 --- a/scio/src/main/scala/org/apache/zeppelin/scio/DisplayHelpers.scala +++ b/scio/src/main/scala/org/apache/zeppelin/scio/DisplayHelpers.scala @@ -35,6 +35,7 @@ private[scio] object DisplayHelpers { private[scio] val tab = "\t" private[scio] val newline = "\n" private[scio] val table = "%table" + private[scio] val endTable = "%text" private[scio] val rowLimitReachedMsg = s"$newline<font color=red>Results are limited to " + maxResults + s" rows.</font>$newline" private[scio] val bQSchemaIncomplete = @@ -52,6 +53,7 @@ private[scio] object DisplayHelpers { println(sCollectionEmptyMsg) } else { println(s"$table value$newline${it.take(maxResults).map(printer).mkString(newline)}") + println(endTable) notifyIfTruncated(it) } } @@ -64,6 +66,7 @@ private[scio] object DisplayHelpers { println(sCollectionEmptyMsg) } else { println(s"$table value$newline${it.take(maxResults).map(printer).mkString(newline)}") + println(endTable) notifyIfTruncated(it) } } @@ -77,6 +80,7 @@ private[scio] object DisplayHelpers { } else { val content = it.take(maxResults).map{ case (k, v) => s"$k$tab$v" }.mkString(newline) println(s"$table key${tab}value$newline$content") + println(endTable) notifyIfTruncated(it) } } @@ -97,6 +101,7 @@ private[scio] object DisplayHelpers { val firstStr = first.productIterator.mkString(tab) val content = it.take(maxResults - 1).map(_.productIterator.mkString(tab)).mkString(newline) println(s"$table $header$newline$firstStr$newline$content") + println(endTable) notifyIfTruncated(it) } } @@ -125,6 +130,7 @@ private[scio] object DisplayHelpers { .map(r => fieldNames.map(r.get).mkString(tab)) .mkString(newline) println(s"$table $header$newline$firstStr$newline$content") + println(endTable) notifyIfTruncated(it) } } @@ -151,6 +157,7 @@ private[scio] object DisplayHelpers { .mkString(newline) println(s"$table $header$newline$content") + println(endTable) notifyIfTruncated(it) } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/82d836fd/scio/src/test/scala/org/apache/zeppelin/scio/DisplayHelpersTest.scala ---------------------------------------------------------------------- diff --git a/scio/src/test/scala/org/apache/zeppelin/scio/DisplayHelpersTest.scala b/scio/src/test/scala/org/apache/zeppelin/scio/DisplayHelpersTest.scala index 1ba4c7e..6dd05ab 100644 --- a/scio/src/test/scala/org/apache/zeppelin/scio/DisplayHelpersTest.scala +++ b/scio/src/test/scala/org/apache/zeppelin/scio/DisplayHelpersTest.scala @@ -48,6 +48,7 @@ class DisplayHelpersTest extends FlatSpec with Matchers { // ----------------------------------------------------------------------------------------------- private val anyValHeader = s"$table value" + private val endTable = DisplayHelpers.endTable "DisplayHelpers" should "support Integer SCollection via AnyVal" in { import org.apache.zeppelin.scio.DisplaySCollectionImplicits.ZeppelinSCollection @@ -59,8 +60,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { o should contain theSameElementsAs Seq(anyValHeader, "1", "2", - "3") + "3", + endTable) o.head should be(anyValHeader) + o.last should be(endTable) } it should "support Long SCollection via AnyVal" in { @@ -73,8 +76,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { o should contain theSameElementsAs Seq(anyValHeader, "1", "2", - "3") + "3", + endTable) o.head should be(anyValHeader) + o.last should be(endTable) } it should "support Double SCollection via AnyVal" in { @@ -87,8 +92,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { o should contain theSameElementsAs Seq(anyValHeader, "1.0", "2.0", - "3.0") + "3.0", + endTable) o.head should be(anyValHeader) + o.last should be(endTable) } it should "support Float SCollection via AnyVal" in { @@ -101,8 +108,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { o should contain theSameElementsAs Seq(anyValHeader, "1.0", "2.0", - "3.0") + "3.0", + endTable) o.head should be(anyValHeader) + o.last should be(endTable) } it should "support Short SCollection via AnyVal" in { @@ -115,8 +124,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { o should contain theSameElementsAs Seq(anyValHeader, "1", "2", - "3") + "3", + endTable) o.head should be(anyValHeader) + o.last should be(endTable) } it should "support Byte SCollection via AnyVal" in { @@ -129,8 +140,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { o should contain theSameElementsAs Seq(anyValHeader, "1", "2", - "3") + "3", + endTable) o.head should be(anyValHeader) + o.last should be(endTable) } it should "support Boolean SCollection via AnyVal" in { @@ -143,8 +156,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { o should contain theSameElementsAs Seq(anyValHeader, "true", "false", - "true") + "true", + endTable) o.head should be(anyValHeader) + o.last should be(endTable) } it should "support Char SCollection via AnyVal" in { @@ -157,8 +172,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { o should contain theSameElementsAs Seq(anyValHeader, "a", "b", - "c") + "c", + endTable) o.head should be(anyValHeader) + o.last should be(endTable) } it should "support SCollection of AnyVal over row limit" in { @@ -199,8 +216,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { o should contain theSameElementsAs Seq(stringHeader, "a", "b", - "c") + "c", + endTable) o.head should be (stringHeader) + o.last should be (endTable) } it should "support empty SCollection of String" in { @@ -240,8 +259,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { } o should contain theSameElementsAs Seq(kvHeader, s"3${tab}4", - s"1${tab}2") + s"1${tab}2", + endTable) o.head should be (kvHeader) + o.last should be (endTable) } it should "support KV (str keys) SCollection" in { @@ -253,8 +274,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { } o should contain theSameElementsAs Seq(kvHeader, s"foo${tab}2", - s"bar${tab}4") + s"bar${tab}4", + endTable) o.head should be (kvHeader) + o.last should be (endTable) } it should "support KV (str values) SCollection" in { @@ -266,8 +289,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { } o should contain theSameElementsAs Seq(kvHeader, s"2${tab}foo", - s"4${tab}bar") + s"4${tab}bar", + endTable) o.head should be (kvHeader) + o.last should be (endTable) } it should "support empty KV SCollection" in { @@ -305,8 +330,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { in.closeAndDisplay() } } - o should contain theSameElementsAs (Seq(tupleHeader) ++ Seq.fill(3)(s"1${tab}2${tab}3")) + o should contain theSameElementsAs + (Seq(tupleHeader, endTable) ++ Seq.fill(3)(s"1${tab}2${tab}3")) o.head should be(tupleHeader) + o.last should be (endTable) } it should "support SCollection of Tuple of 22" in { @@ -318,9 +345,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { in.closeAndDisplay() } } - o should contain theSameElementsAs (Seq(tupleHeader) ++ + o should contain theSameElementsAs (Seq(tupleHeader, endTable) ++ Seq.fill(3)((1 to 21).map(i => s"$i$tab").mkString + "22")) o.head should be(tupleHeader) + o.last should be (endTable) } it should "support SCollection of Case Class of 22" in { @@ -332,9 +360,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { in.closeAndDisplay() } } - o should contain theSameElementsAs (Seq(tupleHeader) ++ + o should contain theSameElementsAs (Seq(tupleHeader, endTable) ++ Seq.fill(3)((1 to 21).map(i => s"$i$tab").mkString + "22")) o.head should be(tupleHeader) + o.last should be (endTable) } it should "support SCollection of Case Class" in { @@ -344,9 +373,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { in.closeAndDisplay() } } - o should contain theSameElementsAs (Seq(testCaseClassHeader) ++ + o should contain theSameElementsAs (Seq(testCaseClassHeader, endTable) ++ Seq.fill(3)(s"1${tab}foo${tab}2.0")) o.head should be(testCaseClassHeader) + o.last should be (endTable) } it should "support empty SCollection of Product" in { @@ -423,9 +453,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { in.closeAndDisplay() } } - o should contain theSameElementsAs (Seq(avroGenericRecordHeader) ++ + o should contain theSameElementsAs (Seq(avroGenericRecordHeader, endTable) ++ Seq.fill(3)(s"1${tab}1.0${tab}user1${tab}checking")) o.head should be(avroGenericRecordHeader) + o.last should be (endTable) } it should "support SCollection of SpecificRecord Avro" in { @@ -436,9 +467,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { in.closeAndDisplay() } } - o should contain theSameElementsAs (Seq(avroAccountHeader) ++ + o should contain theSameElementsAs (Seq(avroAccountHeader, endTable) ++ Seq.fill(3)(s"2${tab}checking${tab}user2${tab}2.0")) o.head should be(avroAccountHeader) + o.last should be (endTable) } it should "support empty SCollection of SpecificRecord Avro" in { @@ -509,9 +541,10 @@ class DisplayHelpersTest extends FlatSpec with Matchers { in.closeAndDisplay(bQSchema) } } - o should contain theSameElementsAs (Seq(bQHeader) ++ + o should contain theSameElementsAs (Seq(bQHeader, endTable) ++ Seq.fill(3)(s"3${tab}3.0${tab}checking${tab}user3")) o.head should be(bQHeader) + o.last should be (endTable) } it should "print error on empty BQ schema" in {
