Skip to content
Snippets Groups Projects
Commit b5e6e8bc authored by Patrick Wendell's avatar Patrick Wendell
Browse files

Cleaning up some code for Job Progress

parent 93e8ed85
No related branches found
No related tags found
No related merge requests found
...@@ -38,7 +38,7 @@ class IndexPage(parent: JobProgressUI) { ...@@ -38,7 +38,7 @@ class IndexPage(parent: JobProgressUI) {
def getElapsedTime(submitted: Option[Long], completed: Long): String = { def getElapsedTime(submitted: Option[Long], completed: Long): String = {
submitted match { submitted match {
case Some(t) => Duration(completed - t, "milliseconds").printHMS case Some(t) => parent.formatDuration(completed - t)
case _ => "Unknown" case _ => "Unknown"
} }
} }
...@@ -53,7 +53,7 @@ class IndexPage(parent: JobProgressUI) { ...@@ -53,7 +53,7 @@ class IndexPage(parent: JobProgressUI) {
case (true, true) => "Read/Write" case (true, true) => "Read/Write"
case (true, false) => "Read" case (true, false) => "Read"
case (false, true) => "Write" case (false, true) => "Write"
case _ => "None" case _ => ""
} }
<tr> <tr>
......
package spark.ui.jobs package spark.ui.jobs
import akka.util.Duration
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletRequest
...@@ -20,16 +22,17 @@ import spark.Success ...@@ -20,16 +22,17 @@ import spark.Success
private[spark] class JobProgressUI(val sc: SparkContext) { private[spark] class JobProgressUI(val sc: SparkContext) {
private var _listener: Option[JobProgressListener] = None private var _listener: Option[JobProgressListener] = None
def listener = _listener.get def listener = _listener.get
val dateFmt = new SimpleDateFormat("EEE, MMM d yyyy HH:mm:ss") val dateFmt = new SimpleDateFormat("EEE, MMM d yyyy HH:mm:ss")
private val indexPage = new IndexPage(this)
private val stagePage = new StagePage(this)
def start() { def start() {
_listener = Some(new JobProgressListener) _listener = Some(new JobProgressListener)
sc.addSparkListener(listener) sc.addSparkListener(listener)
} }
private val indexPage = new IndexPage(this) def formatDuration(ms: Long) = Duration(ms, "milliseconds").printHMS
private val stagePage = new StagePage(this)
def getHandlers = Seq[(String, Handler)]( def getHandlers = Seq[(String, Handler)](
("/stages/stage", (request: HttpServletRequest) => stagePage.render(request)), ("/stages/stage", (request: HttpServletRequest) => stagePage.render(request)),
......
...@@ -8,6 +8,7 @@ import scala.xml.Node ...@@ -8,6 +8,7 @@ import scala.xml.Node
import spark.ui.UIUtils._ import spark.ui.UIUtils._
import spark.util.Distribution import spark.util.Distribution
import spark.Utils
import spark.scheduler.cluster.TaskInfo import spark.scheduler.cluster.TaskInfo
import spark.executor.TaskMetrics import spark.executor.TaskMetrics
...@@ -24,24 +25,30 @@ class StagePage(parent: JobProgressUI) { ...@@ -24,24 +25,30 @@ class StagePage(parent: JobProgressUI) {
val shuffleWrite = listener.hasShuffleWrite(stageId) val shuffleWrite = listener.hasShuffleWrite(stageId)
val taskHeaders: Seq[String] = val taskHeaders: Seq[String] =
Seq("Task ID", "Service Time (ms)", "Locality Level", "Worker", "Launch Time") ++ Seq("Task ID", "Service Time", "Locality Level", "Worker", "Launch Time") ++
{if (shuffleRead) Seq("Shuffle Read (bytes)") else Nil} ++ {if (shuffleRead) Seq("Shuffle Read") else Nil} ++
{if (shuffleWrite) Seq("Shuffle Write (bytes)") else Nil} {if (shuffleWrite) Seq("Shuffle Write") else Nil}
val taskTable = listingTable(taskHeaders, taskRow, tasks) val taskTable = listingTable(taskHeaders, taskRow, tasks)
// TODO(pwendell): Consider factoring this more nicely with the functions in SparkListener
val serviceTimes = tasks.map{case (info, metrics) => metrics.executorRunTime.toDouble} val serviceTimes = tasks.map{case (info, metrics) => metrics.executorRunTime.toDouble}
val serviceQuantiles = "Service Time" +: Distribution(serviceTimes).get.getQuantiles().map(_.toString) val serviceQuantiles = "Service Time" +: Distribution(serviceTimes).get.getQuantiles().map(
ms => parent.formatDuration(ms.toLong))
val shuffleReadSizes = tasks.map{ def getQuantileCols(data: Seq[Double]) =
case(info, metrics) => metrics.shuffleReadMetrics.map(_.remoteBytesRead).getOrElse(0L).toDouble} Distribution(data).get.getQuantiles().map(d => Utils.memoryBytesToString(d.toLong))
val shuffleReadQuantiles = "Shuffle Read" +: Distribution(shuffleReadSizes).get.getQuantiles().map(_.toString)
val shuffleWriteSizes = tasks.map{ val shuffleReadSizes = tasks.map {
case(info, metrics) => metrics.shuffleWriteMetrics.map(_.shuffleBytesWritten).getOrElse(0L).toDouble} case(info, metrics) =>
val shuffleWriteQuantiles = "Shuffle Write" +: Distribution(shuffleWriteSizes).get.getQuantiles().map(_.toString) metrics.shuffleReadMetrics.map(_.remoteBytesRead).getOrElse(0L).toDouble
}
val shuffleReadQuantiles = "Shuffle Read (Remote)" +: getQuantileCols(shuffleReadSizes)
val shuffleWriteSizes = tasks.map {
case(info, metrics) =>
metrics.shuffleWriteMetrics.map(_.shuffleBytesWritten).getOrElse(0L).toDouble
}
val shuffleWriteQuantiles = "Shuffle Write" +: getQuantileCols(shuffleWriteSizes)
val listings: Seq[Seq[String]] = Seq(serviceQuantiles, val listings: Seq[Seq[String]] = Seq(serviceQuantiles,
if (shuffleRead) shuffleReadQuantiles else Nil, if (shuffleRead) shuffleReadQuantiles else Nil,
...@@ -62,12 +69,14 @@ class StagePage(parent: JobProgressUI) { ...@@ -62,12 +69,14 @@ class StagePage(parent: JobProgressUI) {
val (info, metrics) = taskData val (info, metrics) = taskData
<tr> <tr>
<td>{info.taskId}</td> <td>{info.taskId}</td>
<td>{metrics.executorRunTime}</td> <td>{parent.formatDuration(metrics.executorRunTime)}</td>
<td>{info.taskLocality}</td> <td>{info.taskLocality}</td>
<td>{info.hostPort}</td> <td>{info.hostPort}</td>
<td>{dateFmt.format(new Date(info.launchTime))}</td> <td>{dateFmt.format(new Date(info.launchTime))}</td>
{metrics.shuffleReadMetrics.map{m => <td>{m.remoteBytesRead}</td>}.getOrElse("") } {metrics.shuffleReadMetrics.map{m =>
{metrics.shuffleWriteMetrics.map{m => <td>{m.shuffleBytesWritten}</td>}.getOrElse("") } <td>{Utils.memoryBytesToString(m.remoteBytesRead)}</td>}.getOrElse("") }
{metrics.shuffleWriteMetrics.map{m =>
<td>{Utils.memoryBytesToString(m.shuffleBytesWritten)}</td>}.getOrElse("") }
</tr> </tr>
} }
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment