This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 62d6a3b998a [SPARK-42817][CORE] Logging the shuffle service name once
in ApplicationMaster
62d6a3b998a is described below
commit 62d6a3b998a1ad341e4d0870e0cc00d46be96c6e
Author: Chandni Singh <[email protected]>
AuthorDate: Thu Mar 16 14:27:31 2023 -0700
[SPARK-42817][CORE] Logging the shuffle service name once in
ApplicationMaster
### What changes were proposed in this pull request?
Removed the logging of shuffle service name multiple times in the driver
log. It gets logged everytime a new executor is allocated.
### Why are the changes needed?
This is needed because currently the driver logs gets polluted by these
logs:
```
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
22/08/03 20:42:07 INFO ExecutorRunnable: Initializing service data for
shuffle service using name 'spark_shuffle_311'
```
### Does this PR introduce _any_ user-facing change?
Yes, the shuffle service name will be just logged once in the driver.
### How was this patch tested?
Tested manually since it just changes the logging.
With this see this logged in the driver logs:
`23/03/15 16:50:54 INFO ApplicationMaster: Initializing service data for
shuffle service using name 'spark_shuffle_311'`
Closes #40448 from otterc/SPARK-42817.
Authored-by: Chandni Singh <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit f025d5eb1c2c9a6f7933679aa80752e806df9d2a)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 5 ++++-
.../main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala | 1 -
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 9815fa6df8a..73deaf7a028 100644
---
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -498,7 +498,10 @@ private[spark] class ApplicationMaster(
// that when the driver sends an initial executor request (e.g. after an
AM restart),
// the allocator is ready to service requests.
rpcEnv.setupEndpoint("YarnAM", new AMEndpoint(rpcEnv, driverRef))
-
+ if (_sparkConf.get(SHUFFLE_SERVICE_ENABLED)) {
+ logInfo("Initializing service data for shuffle service using name '" +
+ s"${_sparkConf.get(SHUFFLE_SERVICE_NAME)}'")
+ }
allocator.allocateResources()
val ms =
MetricsSystem.createMetricsSystem(MetricsSystemInstances.APPLICATION_MASTER,
sparkConf)
val prefix = _sparkConf.get(YARN_METRICS_NAMESPACE).getOrElse(appId)
diff --git
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 0148b6f3c95..1f3121ed224 100644
---
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -115,7 +115,6 @@ private[yarn] class ExecutorRunnable(
ByteBuffer.allocate(0)
}
val serviceName = sparkConf.get(SHUFFLE_SERVICE_NAME)
- logInfo(s"Initializing service data for shuffle service using name
'$serviceName'")
ctx.setServiceData(Collections.singletonMap(serviceName, secretBytes))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
