This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 399705512a4 [SPARK-44367][SQL][UI] Show error message on UI for each
failed query
399705512a4 is described below
commit 399705512a417de460529843eb047d5c2e8f9e22
Author: Kent Yao <[email protected]>
AuthorDate: Thu Jul 20 13:51:39 2023 +0800
[SPARK-44367][SQL][UI] Show error message on UI for each failed query
### What changes were proposed in this pull request?
This PR adds an 'error message' col to the failed query execution table on
the SQL/DataFrame tab of UI.
### Why are the changes needed?
The SQL tab of UI is not helping to detect SQL errors. This PR will provide
users with a clear understanding of why their queries have failed.
### Does this PR introduce _any_ user-facing change?
SQL tab of UI shows errors for failed queries
### How was this patch tested?
built and tested locally
Closes #41951 from yaooqinn/SPARK-44367.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../main/scala/org/apache/spark/ui/UIUtils.scala | 28 ++++++++++++
.../scala/org/apache/spark/ui/jobs/StagePage.scala | 15 +------
.../org/apache/spark/ui/jobs/StageTable.scala | 18 +-------
.../scala/org/apache/spark/ui/UIUtilsSuite.scala | 24 ++++++++++-
.../apache/spark/sql/execution/SQLExecution.scala | 2 +-
.../spark/sql/execution/ui/AllExecutionsPage.scala | 50 +++++++++++++++++++---
.../hive/thriftserver/ui/ThriftServerPage.scala | 17 --------
7 files changed, 97 insertions(+), 57 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 0ce647d12c5..f0f8cf1310f 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -31,6 +31,7 @@ import scala.util.control.NonFatal
import scala.xml._
import scala.xml.transform.{RewriteRule, RuleTransformer}
+import org.apache.commons.text.StringEscapeUtils
import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap
import org.apache.spark.internal.Logging
@@ -708,4 +709,31 @@ private[spark] object UIUtils extends Logging {
Seq.empty[Node]
}
}
+
+ private final val ERROR_CLASS_REGEX =
"""\[(?<errorClass>[A-Z][A-Z_.]+[A-Z])]""".r
+
+ private def errorSummary(errorMessage: String): (String, Boolean) = {
+ var isMultiline = true
+ val maybeErrorClass =
+
ERROR_CLASS_REGEX.findFirstMatchIn(errorMessage).map(_.group("errorClass"))
+ val errorClassOrBrief = if (maybeErrorClass.nonEmpty &&
maybeErrorClass.get.nonEmpty) {
+ maybeErrorClass.get
+ } else if (errorMessage.indexOf('\n') >= 0) {
+ errorMessage.substring(0, errorMessage.indexOf('\n'))
+ } else if (errorMessage.indexOf(":") >= 0) {
+ errorMessage.substring(0, errorMessage.indexOf(":"))
+ } else {
+ isMultiline = false
+ errorMessage
+ }
+
+ val errorSummary = StringEscapeUtils.escapeHtml4(errorClassOrBrief)
+ (errorSummary, isMultiline)
+ }
+
+ def errorMessageCell(errorMessage: String): Seq[Node] = {
+ val (summary, isMultiline) = errorSummary(errorMessage)
+ val details = detailsUINode(isMultiline, errorMessage)
+ <td>{summary}{details}</td>
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 1934e9e58e6..02aece6e50a 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -696,7 +696,7 @@ private[ui] class TaskPagedTable(
<td>{formatBytes(task.taskMetrics.map(_.memoryBytesSpilled))}</td>
<td>{formatBytes(task.taskMetrics.map(_.diskBytesSpilled))}</td>
}}
- {errorMessageCell(task.errorMessage.getOrElse(""))}
+ {UIUtils.errorMessageCell(task.errorMessage.getOrElse(""))}
</tr>
}
@@ -713,19 +713,6 @@ private[ui] class TaskPagedTable(
private def metricInfo(task: TaskData)(fn: TaskMetrics => Seq[Node]):
Seq[Node] = {
task.taskMetrics.map(fn).getOrElse(Nil)
}
-
- private def errorMessageCell(error: String): Seq[Node] = {
- val isMultiline = error.indexOf('\n') >= 0
- // Display the first line by default
- val errorSummary = StringEscapeUtils.escapeHtml4(
- if (isMultiline) {
- error.substring(0, error.indexOf('\n'))
- } else {
- error
- })
- val details = UIUtils.detailsUINode(isMultiline, error)
- <td>{errorSummary}{details}</td>
- }
}
private[spark] object ApiHelper {
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
index 9e6eb418fe1..9e78f29e92e 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
@@ -24,8 +24,6 @@ import javax.servlet.http.HttpServletRequest
import scala.xml._
-import org.apache.commons.text.StringEscapeUtils
-
import org.apache.spark.status.AppStatusStore
import org.apache.spark.status.api.v1
import org.apache.spark.ui._
@@ -217,7 +215,7 @@ private[ui] class StagePagedTable(
<td>{data.shuffleWriteWithUnit}</td> ++
{
if (isFailedStage) {
- failureReasonHtml(info)
+ UIUtils.errorMessageCell(info.failureReason.getOrElse(""))
} else {
Seq.empty
}
@@ -225,20 +223,6 @@ private[ui] class StagePagedTable(
}
}
- private def failureReasonHtml(s: v1.StageData): Seq[Node] = {
- val failureReason = s.failureReason.getOrElse("")
- val isMultiline = failureReason.indexOf('\n') >= 0
- // Display the first line by default
- val failureReasonSummary = StringEscapeUtils.escapeHtml4(
- if (isMultiline) {
- failureReason.substring(0, failureReason.indexOf('\n'))
- } else {
- failureReason
- })
- val details = UIUtils.detailsUINode(isMultiline, failureReason)
- <td valign="middle">{failureReasonSummary}{details}</td>
- }
-
private def makeDescription(s: v1.StageData, descriptionOption:
Option[String]): Seq[Node] = {
val basePathUri = UIUtils.prependBaseUri(request, basePath)
diff --git a/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala
b/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala
index 9d040bb4e1e..aecd25f6c8d 100644
--- a/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UIUtilsSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.ui
import scala.xml.{Node, Text}
import scala.xml.Utility.trim
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{ErrorMessageFormat, SparkException, SparkFunSuite,
SparkThrowableHelper}
class UIUtilsSuite extends SparkFunSuite {
import UIUtils._
@@ -189,4 +189,26 @@ class UIUtilsSuite extends SparkFunSuite {
assert(generated.sameElements(expected),
s"\n$errorMsg\n\nExpected:\n$expected\nGenerated:\n$generated")
}
+
+ // scalastyle:off line.size.limit
+ test("SPARK-44367: Extract errorClass from errorMsg with errorMessageCell") {
+ val e1 = "Job aborted due to stage failure: Task 0 in stage 1.0 failed 1
times, most recent failure: Lost task 0.0 in stage 1.0 (TID 1) (10.221.98.22
executor driver): org.apache.spark.SparkArithmeticException: [DIVIDE_BY_ZERO]
Division by zero. Use `try_divide` to tolerate divisor being 0 and return NULL
instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass
this error.\n== SQL(line 1, position 8) ==\nselect a/b from src\n
^^^\n\n\tat org.apache.spark.sql. [...]
+ val cell1 = UIUtils.errorMessageCell(e1)
+ assert(cell1 === <td>{"DIVIDE_BY_ZERO"}{UIUtils.detailsUINode(isMultiline
= true, e1)}</td>)
+
+ val e2 = SparkException.internalError("test")
+ val cell2 = UIUtils.errorMessageCell(e2.getMessage)
+ assert(cell2 === <td>{"INTERNAL_ERROR"}{UIUtils.detailsUINode(isMultiline
= true, e2.getMessage)}</td>)
+
+ val e3 = new SparkException(
+ errorClass = "CANNOT_CAST_DATATYPE",
+ messageParameters = Map("sourceType" -> "long", "targetType" -> "int"),
cause = null)
+ val cell3 = UIUtils.errorMessageCell(SparkThrowableHelper.getMessage(e3,
ErrorMessageFormat.PRETTY))
+ assert(cell3 ===
<td>{"CANNOT_CAST_DATATYPE"}{UIUtils.detailsUINode(isMultiline = true,
e3.getMessage)}</td>)
+
+ val e4 = "java.lang.RuntimeException: random text"
+ val cell4 = UIUtils.errorMessageCell(e4)
+ assert(cell4 ===
<td>{"java.lang.RuntimeException"}{UIUtils.detailsUINode(isMultiline = true,
e4)}</td>)
+ }
+ // scalastyle:on line.size.limit
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
index eeca1669e74..68b29e9e216 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
@@ -124,7 +124,7 @@ object SQLExecution {
val endTime = System.nanoTime()
val errorMessage = ex.map {
case e: SparkThrowable =>
- SparkThrowableHelper.getMessage(e, ErrorMessageFormat.MINIMAL)
+ SparkThrowableHelper.getMessage(e, ErrorMessageFormat.PRETTY)
case e =>
// unexpected behavior
SparkThrowableHelper.getMessage(e)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
index a69ca1bbc80..2e088ec8e4b 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
@@ -75,8 +75,16 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends
WebUIPage("") with L
if (running.nonEmpty) {
val runningPageTable =
- executionsTable(request, "running", running.toSeq,
- executionIdToSubExecutions.mapValues(_.toSeq).toMap, currentTime,
true, true, true)
+ executionsTable(
+ request,
+ "running",
+ running.toSeq,
+ executionIdToSubExecutions.mapValues(_.toSeq).toMap,
+ currentTime,
+ showErrorMessage = false,
+ showRunningJobs = true,
+ showSucceededJobs = true,
+ showFailedJobs = true)
_content ++=
<span id="running" class="collapse-aggregated-runningExecutions
collapse-table"
@@ -93,9 +101,16 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends
WebUIPage("") with L
}
if (completed.nonEmpty) {
- val completedPageTable =
- executionsTable(request, "completed", completed.toSeq,
- executionIdToSubExecutions.mapValues(_.toSeq).toMap, currentTime,
false, true, false)
+ val completedPageTable = executionsTable(
+ request,
+ "completed",
+ completed.toSeq,
+ executionIdToSubExecutions.mapValues(_.toSeq).toMap,
+ currentTime,
+ showErrorMessage = false,
+ showRunningJobs = false,
+ showSucceededJobs = true,
+ showFailedJobs = false)
_content ++=
<span id="completed" class="collapse-aggregated-completedExecutions
collapse-table"
@@ -113,8 +128,16 @@ private[ui] class AllExecutionsPage(parent: SQLTab)
extends WebUIPage("") with L
if (failed.nonEmpty) {
val failedPageTable =
- executionsTable(request, "failed", failed.toSeq,
- executionIdToSubExecutions.mapValues(_.toSeq).toMap, currentTime,
false, true, true)
+ executionsTable(
+ request,
+ "failed",
+ failed.toSeq,
+ executionIdToSubExecutions.mapValues(_.toSeq).toMap,
+ currentTime,
+ showErrorMessage = true,
+ showRunningJobs = false,
+ showSucceededJobs = true,
+ showFailedJobs = true)
_content ++=
<span id="failed" class="collapse-aggregated-failedExecutions
collapse-table"
@@ -176,6 +199,7 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends
WebUIPage("") with L
executionData: Seq[SQLExecutionUIData],
executionIdToSubExecutions: Map[Long, Seq[SQLExecutionUIData]],
currentTime: Long,
+ showErrorMessage: Boolean,
showRunningJobs: Boolean,
showSucceededJobs: Boolean,
showFailedJobs: Boolean): Seq[Node] = {
@@ -195,6 +219,7 @@ private[ui] class AllExecutionsPage(parent: SQLTab) extends
WebUIPage("") with L
UIUtils.prependBaseUri(request, parent.basePath),
"SQL", // subPath
currentTime,
+ showErrorMessage,
showRunningJobs,
showSucceededJobs,
showFailedJobs,
@@ -220,6 +245,7 @@ private[ui] class ExecutionPagedTable(
basePath: String,
subPath: String,
currentTime: Long,
+ showErrorMessage: Boolean,
showRunningJobs: Boolean,
showSucceededJobs: Boolean,
showFailedJobs: Boolean,
@@ -287,6 +313,12 @@ private[ui] class ExecutionPagedTable(
} else {
Seq(("Job IDs", true, None))
}
+ } ++ {
+ if (showErrorMessage) {
+ Seq(("Error Message", true, None))
+ } else {
+ Nil
+ }
} ++ {
if (showSubExecutions) {
Seq(("Sub Execution IDs", true, None))
@@ -363,6 +395,9 @@ private[ui] class ExecutionPagedTable(
{jobLinks(executionTableRow.failedJobData)}
</td>
}}
+ {if (showErrorMessage) {
+ UIUtils.errorMessageCell(executionUIData.errorMessage.getOrElse(""))
+ }}
{if (showSubExecutions) {
<td>
{executionLinks(executionTableRow.subExecutionData.map(_.executionUIData.executionId))}
@@ -536,6 +571,7 @@ private[ui] class ExecutionDataSource(
case "Job IDs" | "Succeeded Job IDs" => Ordering by
(_.completedJobData.headOption)
case "Running Job IDs" => Ordering.by(_.runningJobData.headOption)
case "Failed Job IDs" => Ordering.by(_.failedJobData.headOption)
+ case "Error Message" => Ordering.by(_.executionUIData.errorMessage)
case unknownColumn => throw
QueryExecutionErrors.unknownColumnError(unknownColumn)
}
if (desc) {
diff --git
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
index d0378efd646..d47a99466a5 100644
---
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
+++
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/ThriftServerPage.scala
@@ -23,8 +23,6 @@ import javax.servlet.http.HttpServletRequest
import scala.xml.Node
-import org.apache.commons.text.StringEscapeUtils
-
import org.apache.spark.internal.Logging
import org.apache.spark.sql.hive.thriftserver.ui.ToolTips._
import org.apache.spark.ui._
@@ -274,21 +272,6 @@ private[ui] class SqlStatsPagedTable(
</tr>
}
-
- private def errorMessageCell(errorMessage: String): Seq[Node] = {
- val isMultiline = errorMessage.indexOf('\n') >= 0
- val errorSummary = StringEscapeUtils.escapeHtml4(
- if (isMultiline) {
- errorMessage.substring(0, errorMessage.indexOf('\n'))
- } else {
- errorMessage
- })
- val details = detailsUINode(isMultiline, errorMessage)
- <td>
- {errorSummary}{details}
- </td>
- }
-
private def jobURL(request: HttpServletRequest, jobId: String): String =
"%s/jobs/job/?id=%s".format(UIUtils.prependBaseUri(request,
parent.basePath), jobId)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
