Repository: spark Updated Branches: refs/heads/master 261510005 -> 31da755c8
[SPARK-18975][CORE] Add an API to remove SparkListener ## What changes were proposed in this pull request? In current Spark we could add customized SparkListener through `SparkContext#addListener` API, but there's no equivalent API to remove the registered one. In our scenario SparkListener will be added repeatedly accordingly to the changed environment. If lacks the ability to remove listeners, there might be many registered listeners finally, this is unnecessary and potentially affects the performance. So here propose to add an API to remove registered listener. ## How was this patch tested? Add an unit test to verify it. Author: jerryshao <[email protected]> Closes #16382 from jerryshao/SPARK-18975. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/31da755c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/31da755c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/31da755c Branch: refs/heads/master Commit: 31da755c80aed8219c368fd18c72b42e50be46fc Parents: 2615100 Author: jerryshao <[email protected]> Authored: Thu Dec 22 11:18:22 2016 -0800 Committer: Reynold Xin <[email protected]> Committed: Thu Dec 22 11:18:22 2016 -0800 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/SparkContext.scala | 9 +++++++++ .../scala/org/apache/spark/SparkContextSuite.scala | 16 ++++++++++++++++ 2 files changed, 25 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/31da755c/core/src/main/scala/org/apache/spark/SparkContext.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index cae22d7..efb5f9d 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1571,6 +1571,15 @@ class SparkContext(config: SparkConf) extends Logging { listenerBus.addListener(listener) } + /** + * :: DeveloperApi :: + * Deregister the listener from Spark's listener bus. + */ + @DeveloperApi + def removeSparkListener(listener: SparkListenerInterface): Unit = { + listenerBus.removeListener(listener) + } + private[spark] def getExecutorIds(): Seq[String] = { schedulerBackend match { case b: CoarseGrainedSchedulerBackend => http://git-wip-us.apache.org/repos/asf/spark/blob/31da755c/core/src/test/scala/org/apache/spark/SparkContextSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala index c451c59..8fba82d 100644 --- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.TextInputFormat import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat => NewTextInputFormat} import org.scalatest.Matchers._ +import org.apache.spark.scheduler.SparkListener import org.apache.spark.util.Utils class SparkContextSuite extends SparkFunSuite with LocalSparkContext { @@ -451,4 +452,19 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext { sc.stop() } } + + test("register and deregister Spark listener from SparkContext") { + sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local")) + try { + val sparkListener1 = new SparkListener { } + val sparkListener2 = new SparkListener { } + sc.addSparkListener(sparkListener1) + sc.addSparkListener(sparkListener2) + assert(sc.listenerBus.listeners.contains(sparkListener1)) + assert(sc.listenerBus.listeners.contains(sparkListener2)) + sc.removeSparkListener(sparkListener1) + assert(!sc.listenerBus.listeners.contains(sparkListener1)) + assert(sc.listenerBus.listeners.contains(sparkListener2)) + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
