From c529dceaffad0d0eb611ecb70fb5723dab43d0a1 Mon Sep 17 00:00:00 2001
From: Mark Hamstra <markhamstra@gmail.com>
Date: Wed, 25 Dec 2013 12:16:21 -0800
Subject: [PATCH] Avoid a lump of coal (NPE) in JobProgressListener's stocking.
---
.../org/apache/spark/ui/jobs/JobProgressListener.scala | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
index 07a42f0503..2e51dd5a99 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
@@ -146,12 +146,9 @@ private[spark] class JobProgressListener(val sc: SparkContext) extends SparkList
// update duration
y.taskTime += taskEnd.taskInfo.duration
- taskEnd.taskMetrics.shuffleReadMetrics.foreach { shuffleRead =>
- y.shuffleRead += shuffleRead.remoteBytesRead
- }
-
- taskEnd.taskMetrics.shuffleWriteMetrics.foreach { shuffleWrite =>
- y.shuffleWrite += shuffleWrite.shuffleBytesWritten
+ Option(taskEnd.taskMetrics).foreach { taskMetrics =>
+ taskMetrics.shuffleReadMetrics.foreach { y.shuffleRead += _.remoteBytesRead }
+ taskMetrics.shuffleWriteMetrics.foreach { y.shuffleWrite += _.shuffleBytesWritten }
}
}
case _ => {}
--
GitLab