Skip to content
Snippets Groups Projects
Commit f0382007 authored by Patrick Wendell's avatar Patrick Wendell
Browse files

Bug fix for display of shuffle read/write metrics.

This fixes an error where empty cells are missing if a given task
has no shuffle read/write.
parent 622f83ce
No related branches found
No related tags found
No related merge requests found
...@@ -87,7 +87,7 @@ private[spark] class StagePage(parent: JobProgressUI) { ...@@ -87,7 +87,7 @@ private[spark] class StagePage(parent: JobProgressUI) {
{if (hasShuffleWrite) Seq("Shuffle Write") else Nil} ++ {if (hasShuffleWrite) Seq("Shuffle Write") else Nil} ++
Seq("Details") Seq("Details")
val taskTable = listingTable(taskHeaders, taskRow, tasks) val taskTable = listingTable(taskHeaders, taskRow(hasShuffleRead, hasShuffleWrite), tasks)
// Excludes tasks which failed and have incomplete metrics // Excludes tasks which failed and have incomplete metrics
val validTasks = tasks.filter(t => t._1.status == "SUCCESS" && (t._2.isDefined)) val validTasks = tasks.filter(t => t._1.status == "SUCCESS" && (t._2.isDefined))
...@@ -135,7 +135,8 @@ private[spark] class StagePage(parent: JobProgressUI) { ...@@ -135,7 +135,8 @@ private[spark] class StagePage(parent: JobProgressUI) {
} }
def taskRow(taskData: (TaskInfo, Option[TaskMetrics], Option[ExceptionFailure])): Seq[Node] = { def taskRow(shuffleRead: Boolean, shuffleWrite: Boolean)
(taskData: (TaskInfo, Option[TaskMetrics], Option[ExceptionFailure])): Seq[Node] = {
def fmtStackTrace(trace: Seq[StackTraceElement]): Seq[Node] = def fmtStackTrace(trace: Seq[StackTraceElement]): Seq[Node] =
trace.map(e => <span style="display:block;">{e.toString}</span>) trace.map(e => <span style="display:block;">{e.toString}</span>)
val (info, metrics, exception) = taskData val (info, metrics, exception) = taskData
...@@ -154,10 +155,14 @@ private[spark] class StagePage(parent: JobProgressUI) { ...@@ -154,10 +155,14 @@ private[spark] class StagePage(parent: JobProgressUI) {
<td>{info.taskLocality}</td> <td>{info.taskLocality}</td>
<td>{info.hostPort}</td> <td>{info.hostPort}</td>
<td>{dateFmt.format(new Date(info.launchTime))}</td> <td>{dateFmt.format(new Date(info.launchTime))}</td>
{metrics.flatMap{m => m.shuffleReadMetrics}.map{s => {if (shuffleRead) {
<td>{Utils.memoryBytesToString(s.remoteBytesRead)}</td>}.getOrElse("")} <td>{metrics.flatMap{m => m.shuffleReadMetrics}.map{s =>
{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => Utils.memoryBytesToString(s.remoteBytesRead)}.getOrElse("")}</td>
<td>{Utils.memoryBytesToString(s.shuffleBytesWritten)}</td>}.getOrElse("")} }}
{if (shuffleWrite) {
<td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s =>
Utils.memoryBytesToString(s.shuffleBytesWritten)}.getOrElse("")}</td>
}}
<td>{exception.map(e => <td>{exception.map(e =>
<span> <span>
{e.className} ({e.description})<br/> {e.className} ({e.description})<br/>
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment