From dd11e401e45563b4bdc9829f5d23b68dacac8caf Mon Sep 17 00:00:00 2001
From: Charles Allen <charles@allen-net.com>
Date: Wed, 13 Apr 2016 16:02:49 +0100
Subject: [PATCH] [SPARK-14537][CORE] Make TaskSchedulerImpl waiting fail if
context is shut down
This patch makes the postStartHook throw an IllegalStateException if the SparkContext is shutdown while it is waiting for the backend to be ready
Author: Charles Allen <charles@allen-net.com>
Closes #12301 from drcrallen/SPARK-14537.
---
.../scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index daed2ff50e..c3159188d9 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -571,6 +571,11 @@ private[spark] class TaskSchedulerImpl(
return
}
while (!backend.isReady) {
+ // Might take a while for backend to be ready if it is waiting on resources.
+ if (sc.stopped.get) {
+ // For example: the master removes the application for some reason
+ throw new IllegalStateException("Spark context stopped while waiting for backend")
+ }
synchronized {
this.wait(100)
}
--
GitLab