This is an automated email from the ASF dual-hosted git repository.
mridulm80 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 8d85c5a73185 [SPARK-52776][CORE][3.5] Do not split the comm field in
ProcfsMetricsGetter
8d85c5a73185 is described below
commit 8d85c5a731859f7b9bf91ca66e52ab48362845c5
Author: Maxime Xu <ma...@linkedin.com>
AuthorDate: Wed Jul 16 11:19:25 2025 -0500
[SPARK-52776][CORE][3.5] Do not split the comm field in ProcfsMetricsGetter
### What changes were proposed in this pull request?
This is a backport of #51457.
We are fixing an issue in `ProcfsMetricsGetter` when parsing the
`/proc/<pid>/stat` file. The current implementation will split the comm field
by spaces if it contains them, thereby causing subsequent numbers to be
shifted. The comm field, and only the comm field, is in parentheses so we can
resolve this issue by ignoring everything between the first open parenthesis
and last closing parenthesis when splitting the stat file.
### Why are the changes needed?
These changes are needed to prevent a comm field with spaces from causing
incorrect calculations for vmem/rssmem metrics. Please see
[JIRA](https://issues.apache.org/jira/projects/SPARK/issues/SPARK-52776) for
details.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Added a unit test to test for irregular characters in the comm field
### Was this patch authored or co-authored using generative AI tooling?
No
### Original PR Info
Closes #51457 from max2718281/procfs.
Authored-by: Maxime Xu <maxxulinkedin.com>
(cherry picked from commit cf097a5ab7fcffcb64a765db9b8913304340506f)
Closes #51481 from max2718281/procfs-3.5.
Authored-by: Maxime Xu <ma...@linkedin.com>
Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com>
---
.../org/apache/spark/executor/ProcfsMetricsGetter.scala | 9 ++++++++-
core/src/test/resources/ProcfsMetrics/487713/stat | 1 +
.../org/apache/spark/executor/ProcfsMetricsGetterSuite.scala | 12 ++++++++++++
3 files changed, 21 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
index 5448d7da6d6c..bff8007bccf7 100644
--- a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
@@ -176,7 +176,14 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String
= "/proc/") extends L
}
Utils.tryWithResource(openReader) { in =>
val procInfo = in.readLine
- val procInfoSplit = procInfo.split(" ")
+ // The comm field, which is inside parentheses, could contain spaces.
We should not split
+ // by those spaces as doing so could cause the numbers after it to be
shifted.
+ val commStartIndex = procInfo.indexOf('(')
+ val commEndIndex = procInfo.lastIndexOf(')') + 1
+ val pidArray = Array(procInfo.substring(0, commStartIndex).trim)
+ val commArray = Array(procInfo.substring(commStartIndex, commEndIndex))
+ val splitAfterComm = procInfo.substring(commEndIndex).trim.split(" ")
+ val procInfoSplit = pidArray ++ commArray ++ splitAfterComm
val vmem = procInfoSplit(22).toLong
val rssMem = procInfoSplit(23).toLong * pageSize
if (procInfoSplit(1).toLowerCase(Locale.US).contains("java")) {
diff --git a/core/src/test/resources/ProcfsMetrics/487713/stat
b/core/src/test/resources/ProcfsMetrics/487713/stat
new file mode 100644
index 000000000000..63640b58155b
--- /dev/null
+++ b/core/src/test/resources/ProcfsMetrics/487713/stat
@@ -0,0 +1 @@
+487713 ((Executor) task l)) D 474416 474398 474398 0 -1 4194368 5 0 0 0 0
0 0 0 25 5 1 0 1542745216 7469137920 120815 18446744073709551615
104424108929024 104424108932808 140734257079632 0 0 0 4 3 553671884 1 0 0 17 58
0 0 0 0 0 104424108940536 104424108941336 104424532111360 140734257083781
140734257085131 140734257085131 140734257102797 0
\ No newline at end of file
diff --git
a/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
b/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
index d583afdf07c4..bcafe153be0d 100644
---
a/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
+++
b/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
@@ -62,4 +62,16 @@ class ProcfsMetricsGetterSuite extends SparkFunSuite {
assert(r.pythonVmemTotal == 0)
assert(r.pythonRSSTotal == 0)
}
+
+ test("SPARK-52776: Whitespace and parentheses in the comm field") {
+ val p = new ProcfsMetricsGetter(getTestResourcePath("ProcfsMetrics"))
+ var r = ProcfsMetrics(0, 0, 0, 0, 0, 0)
+ r = p.addProcfsMetricsFromOneProcess(r, 487713)
+ assert(r.jvmVmemTotal == 0)
+ assert(r.jvmRSSTotal == 0)
+ assert(r.pythonVmemTotal == 0)
+ assert(r.pythonRSSTotal == 0)
+ assert(r.otherVmemTotal == 7469137920L)
+ assert(r.otherRSSTotal == 494858240)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
