Repository: zeppelin Updated Branches: refs/heads/branch-0.7 d0ee507bc -> 5c8d7902a
[ZEPPELIN-2067] SparkInterpreter prints unnecessary newline ### What is this PR for? Spark interpreter prints unnecessary new line before the evaluation output is printed. See https://github.com/apache/zeppelin/pull/1975#issuecomment-277581660. This PR make SparkInterpreter ignores unnecessary preceding newline from ReplReporter.error() ### What type of PR is it? Bug Fix ### Todos * [x] - Ignore unnecessary preceding newline * [x] - unittest ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-2067 ### How should this be tested? run ``` %spark import java.util.Date import java.net.URL ``` and see if result looks like (no new line in front of each lines) ``` import java.util.Date import java.net.URL ``` ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Lee moon soo <m...@apache.org> Closes #1981 from Leemoonsoo/ZEPPELIN-2067 and squashes the following commits: 1a31a66 [Lee moon soo] add issue id in the comment 23d5d55 [Lee moon soo] Add unittest 6908bdf [Lee moon soo] Ignore preceding newline from scala RepleReporter.error (cherry picked from commit 0ab026e07b7c852454dd2b9a281f81249cf3d52f) Signed-off-by: Mina Lee <mina...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/5c8d7902 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/5c8d7902 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/5c8d7902 Branch: refs/heads/branch-0.7 Commit: 5c8d7902a2b9d6c009a026b7bd6ee3bc46194a9b Parents: d0ee507 Author: Lee moon soo <m...@apache.org> Authored: Tue Feb 7 13:17:55 2017 +0900 Committer: Mina Lee <mina...@apache.org> Committed: Wed Aug 16 14:56:21 2017 +0900 ---------------------------------------------------------------------- .../apache/zeppelin/spark/SparkInterpreter.java | 2 +- .../util/InterpreterOutputStream.java | 30 ++++++++++++++------ .../zeppelin/rest/ZeppelinSparkClusterTest.java | 24 ++++++++++++++++ 3 files changed, 46 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5c8d7902/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 6edf654..e7bf265 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -981,6 +981,7 @@ public class SparkInterpreter extends Interpreter { } private Results.Result interpret(String line) { + out.ignoreLeadingNewLinesFromScalaReporter(); return (Results.Result) Utils.invokeMethod( intp, "interpret", @@ -1231,7 +1232,6 @@ public class SparkInterpreter extends Interpreter { if (varName == null || varName.isEmpty()) { return; } - Object lastObj = null; try { if (Utils.isScala2_10()) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5c8d7902/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java index b6f01b1..6bdc2db 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java @@ -30,6 +30,7 @@ import java.io.IOException; public class InterpreterOutputStream extends LogOutputStream { public static Logger logger; InterpreterOutput interpreterOutput; + boolean ignoreLeadingNewLinesFromScalaReporter = false; public InterpreterOutputStream(Logger logger) { this.logger = logger; @@ -45,6 +46,18 @@ public class InterpreterOutputStream extends LogOutputStream { @Override public void write(int b) throws IOException { + if (ignoreLeadingNewLinesFromScalaReporter && b == '\n') { + StackTraceElement[] stacks = Thread.currentThread().getStackTrace(); + for (StackTraceElement stack : stacks) { + if (stack.getClassName().equals("scala.tools.nsc.interpreter.ReplReporter") && + stack.getMethodName().equals("error")) { + // ignore. Please see ZEPPELIN-2067 + return; + } + } + } else { + ignoreLeadingNewLinesFromScalaReporter = false; + } super.write(b); if (interpreterOutput != null) { interpreterOutput.write(b); @@ -53,17 +66,13 @@ public class InterpreterOutputStream extends LogOutputStream { @Override public void write(byte [] b) throws IOException { - super.write(b); - if (interpreterOutput != null) { - interpreterOutput.write(b); - } + write(b, 0, b.length); } @Override - public void write(byte [] b, int offset, int len) throws IOException { - super.write(b, offset, len); - if (interpreterOutput != null) { - interpreterOutput.write(b, offset, len); + public void write(byte [] b, int off, int len) throws IOException { + for (int i = off; i < len; i++) { + write(b[i]); } } @@ -80,7 +89,6 @@ public class InterpreterOutputStream extends LogOutputStream { } } - @Override public void flush() throws IOException { super.flush(); @@ -88,4 +96,8 @@ public class InterpreterOutputStream extends LogOutputStream { interpreterOutput.flush(); } } + + public void ignoreLeadingNewLinesFromScalaReporter() { + ignoreLeadingNewLinesFromScalaReporter = true; + } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5c8d7902/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index 7affdd8..9bcdbfa 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -77,6 +77,30 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { } @Test + public void scalaOutputTest() throws IOException { + // create new note + Note note = ZeppelinServer.notebook.createNote(anonymous); + Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); + Map config = p.getConfig(); + config.put("enabled", true); + p.setConfig(config); + p.setText("%spark import java.util.Date\n" + + "import java.net.URL\n" + + "println(\"hello\")\n" + ); + p.setAuthenticationInfo(anonymous); + note.run(p.getId()); + waitForFinish(p); + assertEquals(Status.FINISHED, p.getStatus()); + assertEquals("import java.util.Date\n" + + "import java.net.URL\n" + + "hello\n", p.getResult().message().get(0).getData()); + ZeppelinServer.notebook.removeNote(note.getId(), anonymous); + } + + + + @Test public void basicRDDTransformationAndActionTest() throws IOException { // create new note Note note = ZeppelinServer.notebook.createNote(anonymous);
