Repository: zeppelin Updated Branches: refs/heads/master 86bc9332e -> 96d78ee57
[ZEPPELIN-2714] Soft-code Spark UI button visualization ### What is this PR for? When "spark.ui.enabled" property is set to "false" we should not show the Spark UI button. We keep the same behaviour when this property does not exist or when it exists and it's set to true. ### What type of PR is it? [ Improvement] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2714 ### How should this be tested? 1. Open Zeppelin, go to Interpreter > Spark 1. Click spark ui > msg: "No spark application running" 1. Go to a paragraph, run "sc.parallelize(1 to 100).count()", check "SPARK JOB" button 1. Go to Interpreter > Spark > Set "spark.ui.enabled" to "false" 1. Rerun paragraph > "SPARK JOB" button not visible 1. Go to Interpreter > Spark > "spark ui" button not visible Also, test with "spark.ui.enabled"="true" and other workflow combinations ### Questions: * Does the licenses files need update? N * Is there breaking changes for older versions? N * Does this needs documentation? N Author: Nelson Costa <nelson.cost...@gmail.com> Closes #2456 from necosta/zeppelin2714 and squashes the following commits: 278a23e [Nelson Costa] [ZEPPELIN-2714] Fixed broken unit-test edf29cc [Nelson Costa] [ZEPPELIN-2714] Minor final fix 294dea8 [Nelson Costa] [ZEPPELIN-2714] Improvements to change request a85864c [Nelson Costa] [ZEPPELIN-2714] Soft-code Spark UI button visualization Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/96d78ee5 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/96d78ee5 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/96d78ee5 Branch: refs/heads/master Commit: 96d78ee57b5340b2441a1b8fd866d327d998913b Parents: 86bc933 Author: Nelson Costa <nelson.cost...@gmail.com> Authored: Thu Jul 6 15:04:04 2017 +0100 Committer: Felix Cheung <felixche...@apache.org> Committed: Sun Jul 9 00:43:54 2017 -0700 ---------------------------------------------------------------------- docs/usage/rest_api/interpreter.md | 23 ++++++++++++ .../apache/zeppelin/spark/SparkInterpreter.java | 37 ++++++++++++------- .../zeppelin/rest/InterpreterRestApi.java | 23 ++++-------- .../zeppelin/rest/InterpreterRestApiTest.java | 39 +++++++++++++++++--- .../app/interpreter/interpreter.controller.js | 13 +++++-- 5 files changed, 96 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/docs/usage/rest_api/interpreter.md ---------------------------------------------------------------------- diff --git a/docs/usage/rest_api/interpreter.md b/docs/usage/rest_api/interpreter.md index 4869866..23a7c66 100644 --- a/docs/usage/rest_api/interpreter.md +++ b/docs/usage/rest_api/interpreter.md @@ -604,3 +604,26 @@ The role of registered interpreters, settings and interpreters group are describ </td> </table> +<br/> +### Get interpreter settings metadata info + + <table class="table-configuration"> + <col width="200"> + <tr> + <td>Description</td> + <td>This ```GET``` method returns interpreter settings metadata info. </td> + </tr> + <tr> + <td>URL</td> + <td>```http://[zeppelin-server]:[zeppelin-port]/api/interpreter/metadata/[setting ID]```</td> + </tr> + <tr> + <td>Success code</td> + <td>200</td> + </tr> + <tr> + <td>Fail code</td> + <td> 500 </td> + </tr> + </table> + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index c170e4e..dd1fa11 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -184,11 +184,14 @@ public class SparkInterpreter extends Interpreter { super.onJobStart(jobStart); int jobId = jobStart.jobId(); String jobGroupId = jobStart.properties().getProperty("spark.jobGroup.id"); + String uiEnabled = jobStart.properties().getProperty("spark.ui.enabled"); String jobUrl = getJobUrl(jobId); String noteId = Utils.getNoteId(jobGroupId); String paragraphId = Utils.getParagraphId(jobGroupId); - - if (jobUrl != null && noteId != null && paragraphId != null) { + // Button visible if Spark UI property not set, set as invalid boolean or true + java.lang.Boolean showSparkUI = + uiEnabled == null || !uiEnabled.trim().toLowerCase().equals("false"); + if (showSparkUI && jobUrl != null) { RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient(); Map<String, String> infos = new java.util.HashMap<>(); infos.put("jobUrl", jobUrl); @@ -1040,21 +1043,29 @@ public class SparkInterpreter extends Interpreter { } public void populateSparkWebUrl(InterpreterContext ctx) { - if (sparkUrl == null) { - sparkUrl = getSparkUIUrl(); - Map<String, String> infos = new java.util.HashMap<>(); - if (sparkUrl != null) { - infos.put("url", sparkUrl); - if (ctx != null && ctx.getClient() != null) { - logger.info("Sending metainfos to Zeppelin server: {}", infos.toString()); - getZeppelinContext().setEventClient(ctx.getClient()); - ctx.getClient().onMetaInfosReceived(infos); - } + sparkUrl = getSparkUIUrl(); + Map<String, String> infos = new java.util.HashMap<>(); + infos.put("url", sparkUrl); + String uiEnabledProp = property.getProperty("spark.ui.enabled", "true"); + java.lang.Boolean uiEnabled = java.lang.Boolean.parseBoolean( + uiEnabledProp.trim()); + if (!uiEnabled) { + infos.put("message", "Spark UI disabled"); + } else { + if (StringUtils.isNotBlank(sparkUrl)) { + infos.put("message", "Spark UI enabled"); + } else { + infos.put("message", "No spark url defined"); } } + if (ctx != null && ctx.getClient() != null) { + logger.info("Sending metadata to Zeppelin server: {}", infos.toString()); + getZeppelinContext().setEventClient(ctx.getClient()); + ctx.getClient().onMetaInfosReceived(infos); + } } - public List<File> currentClassPath() { + private List<File> currentClassPath() { List<File> paths = classPath(Thread.currentThread().getContextClassLoader()); String[] cps = System.getProperty("java.class.path").split(File.pathSeparator); if (cps != null) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 17c4027..cd0210e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -246,28 +246,19 @@ public class InterpreterRestApi { } /** - * get the metainfo property value + * get metadata values */ @GET - @Path("getmetainfos/{settingId}") + @Path("metadata/{settingId}") + @ZeppelinApi public Response getMetaInfo(@Context HttpServletRequest req, @PathParam("settingId") String settingId) { - String propName = req.getParameter("propName"); - if (propName == null) { - return new JsonResponse<>(Status.BAD_REQUEST).build(); - } - String propValue = null; InterpreterSetting interpreterSetting = interpreterSettingManager.get(settingId); - Map<String, String> infos = interpreterSetting.getInfos(); - if (infos != null) { - propValue = infos.get(propName); + if (interpreterSetting == null) { + return new JsonResponse<>(Status.NOT_FOUND).build(); } - Map<String, String> respMap = new HashMap<>(); - respMap.put(propName, propValue); - logger.debug("Get meta info"); - logger.debug("Interpretersetting Id: {}, property Name:{}, property value: {}", settingId, - propName, propValue); - return new JsonResponse<>(Status.OK, respMap).build(); + Map<String, String> infos = interpreterSetting.getInfos(); + return new JsonResponse<>(Status.OK, "metadata", infos).build(); } /** http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java index 5333385..28541bd 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java @@ -54,8 +54,8 @@ import static org.hamcrest.MatcherAssert.assertThat; */ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class InterpreterRestApiTest extends AbstractTestRestApi { - Gson gson = new Gson(); - AuthenticationInfo anonymous; + private Gson gson = new Gson(); + private AuthenticationInfo anonymous; @BeforeClass public static void init() throws Exception { @@ -365,21 +365,48 @@ public class InterpreterRestApiTest extends AbstractTestRestApi { delete.releaseConnection(); } - public JsonObject getBodyFieldFromResponse(String rawResponse) { + @Test + public void testGetMetadataInfo() throws IOException { + String jsonRequest = "{\"name\":\"spark\",\"group\":\"spark\"," + + "\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," + + "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," + + "\"dependencies\":[]," + + "\"option\": { \"remote\": true, \"session\": false }}"; + PostMethod post = httpPost("/interpreter/setting/", jsonRequest); + InterpreterSetting created = convertResponseToInterpreterSetting(post.getResponseBodyAsString()); + String settingId = created.getId(); + Map<String, String> infos = new java.util.HashMap<>(); + infos.put("key1", "value1"); + infos.put("key2", "value2"); + ZeppelinServer.notebook.getInterpreterSettingManager().get(settingId).setInfos(infos); + GetMethod get = httpGet("/interpreter/metadata/" + settingId); + assertThat(get, isAllowed()); + JsonObject body = getBodyFieldFromResponse(get.getResponseBodyAsString()); + assertEquals(body.entrySet().size(), infos.size()); + java.util.Map.Entry<String, JsonElement> item = body.entrySet().iterator().next(); + if (item.getKey().equals("key1")) { + assertEquals(item.getValue().getAsString(), "value1"); + } else { + assertEquals(item.getValue().getAsString(), "value2"); + } + get.releaseConnection(); + } + + private JsonObject getBodyFieldFromResponse(String rawResponse) { JsonObject response = gson.fromJson(rawResponse, JsonElement.class).getAsJsonObject(); return response.getAsJsonObject("body"); } - public JsonArray getArrayBodyFieldFromResponse(String rawResponse) { + private JsonArray getArrayBodyFieldFromResponse(String rawResponse) { JsonObject response = gson.fromJson(rawResponse, JsonElement.class).getAsJsonObject(); return response.getAsJsonArray("body"); } - public InterpreterSetting convertResponseToInterpreterSetting(String rawResponse) { + private InterpreterSetting convertResponseToInterpreterSetting(String rawResponse) { return gson.fromJson(getBodyFieldFromResponse(rawResponse), InterpreterSetting.class); } - public static String getSimulatedMarkdownResult(String markdown) { + private static String getSimulatedMarkdownResult(String markdown) { return String.format("<div class=\"markdown-body\">\n<p>%s</p>\n</div>", markdown); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/96d78ee5/zeppelin-web/src/app/interpreter/interpreter.controller.js ---------------------------------------------------------------------- diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index e75740f..dc3619e 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -754,16 +754,21 @@ function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeou } $scope.showSparkUI = function (settingId) { - $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/getmetainfos/' + settingId + '?propName=url') + $http.get(baseUrlSrv.getRestApiBase() + '/interpreter/metadata/' + settingId) .success(function (data, status, headers, config) { - let url = data.body.url - if (!url) { + if (data.body === undefined) { BootstrapDialog.alert({ message: 'No spark application running' }) return } - window.open(url, '_blank') + if (data.body.url) { + window.open(data.body.url, '_blank') + } else { + BootstrapDialog.alert({ + message: data.body.message + }) + } }).error(function (data, status, headers, config) { console.log('Error %o %o', status, data.message) })
