Repository: zeppelin Updated Branches: refs/heads/master 85c525200 -> 61b7162fb
[ZEPPELIN-2257] notification about incompleteness of data ### What is this PR for? Added notification about exceeding the limit in the result. ### What type of PR is it? Improvement ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-2257 ### How should this be tested? 1. Create table *test* with more than 2 records (if not exists) 2. Set parameter `common.max_count ` = 2 3. Execute ``` %jdbc select *from test ``` You should see message about exceeds limit ### Screenshots (if appropriate)  ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Tinkoff DWH <tinkoff....@gmail.com> Closes #2134 from tinkoff-dwh/ZEPPELIN-2257 and squashes the following commits: 367ac65 [Tinkoff DWH] [ZEPPELIN-2257] merge 1048214 [Tinkoff DWH] Merge remote-tracking branch 'origin/master' into ZEPPELIN-2257 3c52b52 [Tinkoff DWH] [ZEPPELIN-2257] custom css class for alert 2e6c976 [Tinkoff DWH] [ZEPPELIN-2257] close button to alert d6dbe3e [Tinkoff DWH] Merge remote-tracking branch 'origin/master' into ZEPPELIN-2257 edeca0e [Tinkoff DWH] [ZEPPELIN-2257] notifications about incompleteness of data Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/61b7162f Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/61b7162f Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/61b7162f Branch: refs/heads/master Commit: 61b7162fb850c682d8f3f6467816cc17a95f65f9 Parents: 85c5252 Author: Tinkoff DWH <tinkoff....@gmail.com> Authored: Mon Apr 3 15:33:50 2017 +0500 Committer: Lee moon soo <m...@apache.org> Committed: Thu Apr 6 08:05:29 2017 +0900 ---------------------------------------------------------------------- .../apache/zeppelin/jdbc/JDBCInterpreter.java | 12 +++-- .../zeppelin/jdbc/JDBCInterpreterTest.java | 2 + .../zeppelin/livy/LivySparkSQLInterpreter.java | 4 +- .../zeppelin/pig/PigQueryInterpreter.java | 8 ++-- .../zeppelin/pig/PigQueryInterpreterTest.java | 2 +- .../zeppelin/spark/SparkSqlInterpreter.java | 7 ++- .../apache/zeppelin/spark/ZeppelinContext.java | 5 ++- .../zeppelin/spark/SparkSqlInterpreterTest.java | 19 +++++++- .../zeppelin/interpreter/InterpreterOutput.java | 6 +-- .../zeppelin/interpreter/InterpreterResult.java | 4 ++ .../zeppelin/interpreter/ResultMessages.java | 46 ++++++++++++++++++++ .../interpreter/InterpreterOutputTest.java | 6 ++- .../app/notebook/paragraph/result/result.css | 12 +++-- 13 files changed, 111 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java ---------------------------------------------------------------------- diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index 1080c2b..47cdfcc 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -47,6 +47,7 @@ import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.ResultMessages; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.jdbc.security.JDBCSecurityImpl; import org.apache.zeppelin.scheduler.Scheduler; @@ -474,7 +475,7 @@ public class JDBCInterpreter extends Interpreter { msg.append(NEWLINE); int displayRowCount = 0; - while (resultSet.next() && displayRowCount < getMaxResult()) { + while (displayRowCount < getMaxResult() && resultSet.next()) { for (int i = 1; i < md.getColumnCount() + 1; i++) { Object resultObject; String resultValue; @@ -602,8 +603,13 @@ public class JDBCInterpreter extends Interpreter { interpreterResult.add(InterpreterResult.Type.TEXT, "Query executed successfully."); } else { - interpreterResult.add( - getResults(resultSet, !containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE))); + String results = getResults(resultSet, + !containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE)); + interpreterResult.add(results); + if (resultSet.next()) { + interpreterResult.add(ResultMessages.getExceedsLimitRowsMessage(getMaxResult(), + String.format("%s.%s", COMMON_KEY, MAX_LINE_KEY))); + } } } else { // Response contains either an update count or there are no results. http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java ---------------------------------------------------------------------- diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java index 2e7e1a5..dc0463a 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java @@ -254,6 +254,8 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter { assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code()); assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType()); assertEquals("ID\tNAME\na\ta_name\n", interpreterResult.message().get(0).getData()); + assertEquals(InterpreterResult.Type.HTML, interpreterResult.message().get(1).getType()); + assertTrue(interpreterResult.message().get(1).getData().contains("alert-warning")); } @Test http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java ---------------------------------------------------------------------- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index 2eaf79c..cdd4eac 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -141,8 +141,8 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter { List<String> rows = parseSQLOutput(message.getData()); result2.add(InterpreterResult.Type.TABLE, StringUtils.join(rows, "\n")); if (rows.size() >= (maxResult + 1)) { - result2.add(InterpreterResult.Type.HTML, - "<font color=red>Results are limited by " + maxResult + ".</font>"); + result2.add(ResultMessages.getExceedsLimitRowsMessage(maxResult, + ZEPPELIN_LIVY_SPARK_SQL_MAX_RESULT)); } } else { result2.add(message.getType(), message.getData()); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java ---------------------------------------------------------------------- diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java index 566b536..385ff45 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java @@ -45,6 +45,7 @@ import java.util.Properties; public class PigQueryInterpreter extends BasePigInterpreter { private static Logger LOGGER = LoggerFactory.getLogger(PigQueryInterpreter.class); + private static final String MAX_RESULTS = "zeppelin.pig.maxResult"; private PigServer pigServer; private int maxResult; @@ -55,7 +56,7 @@ public class PigQueryInterpreter extends BasePigInterpreter { @Override public void open() { pigServer = getPigInterpreter().getPigServer(); - maxResult = Integer.parseInt(getProperty("zeppelin.pig.maxResult")); + maxResult = Integer.parseInt(getProperty(MAX_RESULTS)); } @Override @@ -104,7 +105,7 @@ public class PigQueryInterpreter extends BasePigInterpreter { Iterator<Tuple> iter = pigServer.openIterator(alias); boolean firstRow = true; int index = 0; - while (iter.hasNext() && index <= maxResult) { + while (iter.hasNext() && index < maxResult) { index++; Tuple tuple = iter.next(); if (firstRow && !schemaKnown) { @@ -118,7 +119,8 @@ public class PigQueryInterpreter extends BasePigInterpreter { resultBuilder.append("\n"); } if (index >= maxResult && iter.hasNext()) { - resultBuilder.append("\n<font color=red>Results are limited by " + maxResult + ".</font>"); + resultBuilder.append("\n"); + resultBuilder.append(ResultMessages.getExceedsLimitRowsMessage(maxResult, MAX_RESULTS)); } } catch (IOException e) { // Extract error in the following order http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java ---------------------------------------------------------------------- diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java index aa6bb08..de297c7 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java @@ -153,6 +153,6 @@ public class PigQueryInterpreterTest { assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains("id\n0\n1\n2")); - assertTrue(result.message().get(0).getData().contains("Results are limited by 20")); + assertTrue(result.message().get(1).getData().contains("alert-warning")); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index d42eb50..d2de9a1 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -42,7 +42,10 @@ import org.slf4j.LoggerFactory; * Spark SQL interpreter for Zeppelin. */ public class SparkSqlInterpreter extends Interpreter { - Logger logger = LoggerFactory.getLogger(SparkSqlInterpreter.class); + private Logger logger = LoggerFactory.getLogger(SparkSqlInterpreter.class); + + public static final String MAX_RESULTS = "zeppelin.spark.maxResult"; + AtomicInteger num = new AtomicInteger(0); private int maxResult; @@ -53,7 +56,7 @@ public class SparkSqlInterpreter extends Interpreter { @Override public void open() { - this.maxResult = Integer.parseInt(getProperty("zeppelin.spark.maxResult")); + this.maxResult = Integer.parseInt(getProperty(MAX_RESULTS)); } private SparkInterpreter getSparkInterpreter() { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index d62b68e..6e96d9d 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -46,6 +46,7 @@ import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterHookRegistry; import org.apache.zeppelin.interpreter.RemoteWorksController; +import org.apache.zeppelin.interpreter.ResultMessages; import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper; import org.apache.zeppelin.spark.dep.SparkDependencyResolver; import org.apache.zeppelin.resource.Resource; @@ -295,9 +296,9 @@ public class ZeppelinContext { } if (rows.length > maxResult) { - msg.append("<!--TABLE_COMMENT-->"); msg.append("\n"); - msg.append("<font color=red>Results are limited by " + maxResult + ".</font>"); + msg.append(ResultMessages.getExceedsLimitRowsMessage(maxResult, + SparkSqlInterpreter.MAX_RESULTS)); } sc.clearJobGroup(); return msg.toString(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java ---------------------------------------------------------------------- diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java index 5984645..ebb5e9a 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java @@ -47,7 +47,7 @@ public class SparkSqlInterpreterTest { public static void setUp() throws Exception { Properties p = new Properties(); p.putAll(SparkInterpreterTest.getSparkTestProperties(tmpDir)); - p.setProperty("zeppelin.spark.maxResult", "1000"); + p.setProperty("zeppelin.spark.maxResult", "10"); p.setProperty("zeppelin.spark.concurrentSQL", "false"); p.setProperty("zeppelin.spark.sql.stacktrace", "false"); @@ -160,4 +160,21 @@ public class SparkSqlInterpreterTest { assertEquals(Type.TABLE, ret.message().get(0).getType()); assertEquals("name\tage\ngates\tnull\n", ret.message().get(0).getData()); } + + @Test + public void testMaxResults() { + repl.interpret("case class P(age:Int)", context); + repl.interpret( + "val gr = sc.parallelize(Seq(P(1),P(2),P(3),P(4),P(5),P(6),P(7),P(8),P(9),P(10),P(11)))", + context); + if (isDataFrameSupported()) { + repl.interpret("gr.toDF.registerTempTable(\"gr\")", context); + } else { + repl.interpret("gr.registerTempTable(\"gr\")", context); + } + + InterpreterResult ret = sql.interpret("select * from gr", context); + assertEquals(InterpreterResult.Code.SUCCESS, ret.code()); + assertTrue(ret.message().get(1).getData().contains("alert-warning")); + } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java index bf0d4b6..c3d25c9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java @@ -184,9 +184,9 @@ public class InterpreterOutput extends OutputStream { if (b == NEW_LINE_CHAR && currentOut != null) { InterpreterResult.Type type = currentOut.getType(); if (type == InterpreterResult.Type.TEXT || type == InterpreterResult.Type.TABLE) { - - setType(InterpreterResult.Type.TEXT); - getCurrentOutput().write("Output exceeds " + limit + ". Truncated.\n"); + setType(InterpreterResult.Type.HTML); + getCurrentOutput().write(ResultMessages.getExceedsLimitSizeMessage(limit, + "ZEPPELIN_INTERPRETER_OUTPUT_LIMIT").getData().getBytes()); truncated = true; return; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java index 5288f6f..2316490 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java @@ -96,6 +96,10 @@ public class InterpreterResult implements Serializable { msg.add(new InterpreterResultMessage(type, data)); } + public void add(InterpreterResultMessage interpreterResultMessage) { + msg.add(interpreterResultMessage); + } + public Code code() { return code; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ResultMessages.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ResultMessages.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ResultMessages.java new file mode 100644 index 0000000..d32299e --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/ResultMessages.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter; + +/** + * + */ +public class ResultMessages { + public static final String EXCEEDS_LIMIT_ROWS = + "<strong>Output is truncated</strong> to %s rows. Learn more about <strong>%s</strong>"; + public static final String EXCEEDS_LIMIT_SIZE = + "<strong>Output is truncated</strong> to %s bytes. Learn more about <strong>%s</strong>"; + public static final String EXCEEDS_LIMIT = + "<div class=\"result-alert alert-warning\" role=\"alert\">" + + "<button type=\"button\" class=\"close\" data-dismiss=\"alert\" aria-label=\"Close\">" + + "<span aria-hidden=\"true\">×</span></button>" + + "%s" + + "</div>"; + + public static InterpreterResultMessage getExceedsLimitRowsMessage(int amount, String variable) { + InterpreterResultMessage message = new InterpreterResultMessage(InterpreterResult.Type.HTML, + String.format(EXCEEDS_LIMIT, String.format(EXCEEDS_LIMIT_ROWS, amount, variable))); + return message; + } + + public static InterpreterResultMessage getExceedsLimitSizeMessage(int amount, String variable) { + InterpreterResultMessage message = new InterpreterResultMessage(InterpreterResult.Type.HTML, + String.format(EXCEEDS_LIMIT, String.format(EXCEEDS_LIMIT_SIZE, amount, variable))); + return message; + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputTest.java index 021edce..82d8c3f 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputTest.java @@ -171,13 +171,15 @@ public class InterpreterOutputTest implements InterpreterOutputListener { // truncate text out.write("%text hello\nworld\n"); assertEquals("hello", new String(out.getOutputAt(0).toByteArray())); - assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("Truncated")); + out.getOutputAt(1).flush(); + assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("truncated")); // truncate table out = new InterpreterOutput(this); out.write("%table key\tvalue\nhello\t100\nworld\t200\n"); assertEquals("key\tvalue", new String(out.getOutputAt(0).toByteArray())); - assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("Truncated")); + out.getOutputAt(1).flush(); + assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("truncated")); // does not truncate html out = new InterpreterOutput(this); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/61b7162f/zeppelin-web/src/app/notebook/paragraph/result/result.css ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/notebook/paragraph/result/result.css b/zeppelin-web/src/app/notebook/paragraph/result/result.css index 905b88c..97eab53 100644 --- a/zeppelin-web/src/app/notebook/paragraph/result/result.css +++ b/zeppelin-web/src/app/notebook/paragraph/result/result.css @@ -13,9 +13,9 @@ */ .result-chart-selector { - margin-bottom: 10px; - position: relative; - display: inline-block; + margin-bottom: 10px; + position: relative; + display: inline-block; vertical-align: middle; } @@ -40,3 +40,9 @@ -moz-transform: rotate(90deg) scaleX(-1); -ms-transform: rotate(90deg) scaleX(-1); } + +.result-alert { + padding: 15px; + border: 1px solid transparent; + border-radius: 4px; +}
