Skip to content
Snippets Groups Projects
Commit a88aa5e6 authored by Kay Ousterhout's avatar Kay Ousterhout
Browse files

Fixed 2 bugs in executor UI.

1) UI crashed if the executor UI was loaded before any tasks started.
2) The total tasks was incorrectly reported due to using string (rather
than int) arithmetic.
parent 3f14cbab
No related branches found
No related tags found
No related merge requests found
...@@ -38,10 +38,9 @@ private[spark] class ExecutorsUI(val sc: SparkContext) { ...@@ -38,10 +38,9 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
def render(request: HttpServletRequest): Seq[Node] = { def render(request: HttpServletRequest): Seq[Node] = {
val storageStatusList = sc.getExecutorStorageStatus val storageStatusList = sc.getExecutorStorageStatus
val maxMem = storageStatusList.map(_.maxMem).reduce(_+_) val maxMem = storageStatusList.map(_.maxMem).fold(0L)(_+_)
val memUsed = storageStatusList.map(_.memUsed()).reduce(_+_) val memUsed = storageStatusList.map(_.memUsed()).fold(0L)(_+_)
val diskSpaceUsed = storageStatusList.flatMap(_.blocks.values.map(_.diskSize)) val diskSpaceUsed = storageStatusList.flatMap(_.blocks.values.map(_.diskSize)).fold(0L)(_+_)
.reduceOption(_+_).getOrElse(0L)
val execHead = Seq("Executor ID", "Address", "RDD blocks", "Memory used", "Disk used", val execHead = Seq("Executor ID", "Address", "RDD blocks", "Memory used", "Disk used",
"Active tasks", "Failed tasks", "Complete tasks", "Total tasks") "Active tasks", "Failed tasks", "Complete tasks", "Total tasks")
...@@ -93,10 +92,9 @@ private[spark] class ExecutorsUI(val sc: SparkContext) { ...@@ -93,10 +92,9 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
val memUsed = sc.getExecutorStorageStatus(a).memUsed().toString val memUsed = sc.getExecutorStorageStatus(a).memUsed().toString
val maxMem = sc.getExecutorStorageStatus(a).maxMem.toString val maxMem = sc.getExecutorStorageStatus(a).maxMem.toString
val diskUsed = sc.getExecutorStorageStatus(a).diskUsed().toString val diskUsed = sc.getExecutorStorageStatus(a).diskUsed().toString
val activeTasks = listener.executorToTasksActive.get(a.toString).map(l => l.size) val activeTasks = listener.executorToTasksActive.get(a.toString).map(l => l.size).getOrElse(0)
.getOrElse(0).toString val failedTasks = listener.executorToTasksFailed.getOrElse(a.toString, 0)
val failedTasks = listener.executorToTasksFailed.getOrElse(a.toString, 0).toString val completedTasks = listener.executorToTasksComplete.getOrElse(a.toString, 0)
val completedTasks = listener.executorToTasksComplete.getOrElse(a.toString, 0).toString
val totalTasks = activeTasks + failedTasks + completedTasks val totalTasks = activeTasks + failedTasks + completedTasks
Seq( Seq(
...@@ -106,10 +104,10 @@ private[spark] class ExecutorsUI(val sc: SparkContext) { ...@@ -106,10 +104,10 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
memUsed, memUsed,
maxMem, maxMem,
diskUsed, diskUsed,
activeTasks, activeTasks.toString,
failedTasks, failedTasks.toString,
completedTasks, completedTasks.toString,
totalTasks totalTasks.toString
) )
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment