diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index be3dac4d2408455d1333b6c1a0ed845146e3d68b..47174e4efee81757a28f879410d7764224607bdb 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -114,4 +114,9 @@ package object config {
private[spark] val PYSPARK_PYTHON = ConfigBuilder("spark.pyspark.python")
.stringConf
.createOptional
+
+ // To limit memory usage, we only track information for a fixed number of tasks
+ private[spark] val UI_RETAINED_TASKS = ConfigBuilder("spark.ui.retainedTasks")
+ .intConf
+ .createWithDefault(100000)
}
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
index 491f7160bc6ac0b9b8ee74ff83fac7de7716a888..d3a4f9d3223a7e553274ebc3323bd8fae21e52d6 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
@@ -19,12 +19,13 @@ package org.apache.spark.ui.jobs
import java.util.concurrent.TimeoutException
-import scala.collection.mutable.{HashMap, HashSet, ListBuffer}
+import scala.collection.mutable.{HashMap, HashSet, LinkedHashMap, ListBuffer}
import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.executor.TaskMetrics
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config._
import org.apache.spark.scheduler._
import org.apache.spark.scheduler.SchedulingMode.SchedulingMode
import org.apache.spark.storage.BlockManagerId
@@ -93,6 +94,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
val retainedStages = conf.getInt("spark.ui.retainedStages", SparkUI.DEFAULT_RETAINED_STAGES)
val retainedJobs = conf.getInt("spark.ui.retainedJobs", SparkUI.DEFAULT_RETAINED_JOBS)
+ val retainedTasks = conf.get(UI_RETAINED_TASKS)
// We can test for memory leaks by ensuring that collections that track non-active jobs and
// stages do not grow without bound and that collections for active jobs/stages eventually become
@@ -405,6 +407,11 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
taskData.updateTaskMetrics(taskMetrics)
taskData.errorMessage = errorMessage
+ // If Tasks is too large, remove and garbage collect old tasks
+ if (stageData.taskData.size > retainedTasks) {
+ stageData.taskData = stageData.taskData.drop(stageData.taskData.size - retainedTasks)
+ }
+
for (
activeJobsDependentOnStage <- stageIdToActiveJobIds.get(taskEnd.stageId);
jobId <- activeJobsDependentOnStage;
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index ea7acc4734dff2eb6c7a1ea807c6f55dc71f7318..a266164587e4e763c1c199af6b3a42b3d2e634d0 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -133,7 +133,14 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val stageData = stageDataOption.get
val tasks = stageData.taskData.values.toSeq.sortBy(_.taskInfo.launchTime)
- val numCompleted = tasks.count(_.taskInfo.finished)
+ val numCompleted = stageData.numCompleteTasks
+ val totalTasks = stageData.numActiveTasks +
+ stageData.numCompleteTasks + stageData.numFailedTasks
+ val totalTasksNumStr = if (totalTasks == tasks.size) {
+ s"$totalTasks"
+ } else {
+ s"$totalTasks, showing ${tasks.size}"
+ }
val allAccumulables = progressListener.stageIdToData((stageId, stageAttemptId)).accumulables
val externalAccumulables = allAccumulables.values.filter { acc => !acc.internal }
@@ -591,7 +598,8 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
<div>{summaryTable.getOrElse("No tasks have reported metrics yet.")}</div> ++
aggMetrics ++
maybeAccumulableTable ++
- <h4 id="tasks-section">Tasks</h4> ++ taskTableHTML ++ jsForScrollingDownToTaskTable
+ <h4 id="tasks-section">Tasks ({totalTasksNumStr})</h4> ++
+ taskTableHTML ++ jsForScrollingDownToTaskTable
UIUtils.headerSparkPage(stageHeader, content, parent, showVisualization = true)
}
}
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
index 20dde7cec827ed294c7109b311be882109191c28..66b88129ee414184efb51e32141822b09ab30c05 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ui.jobs
import scala.collection.mutable
-import scala.collection.mutable.HashMap
+import scala.collection.mutable.{HashMap, LinkedHashMap}
import org.apache.spark.JobExecutionStatus
import org.apache.spark.executor.{ShuffleReadMetrics, ShuffleWriteMetrics, TaskMetrics}
@@ -97,7 +97,7 @@ private[spark] object UIData {
var description: Option[String] = None
var accumulables = new HashMap[Long, AccumulableInfo]
- var taskData = new HashMap[Long, TaskUIData]
+ var taskData = new LinkedHashMap[Long, TaskUIData]
var executorSummary = new HashMap[String, ExecutorSummary]
def hasInput: Boolean = inputBytes > 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
index 11eec0b49c40b16f5024d2540d0886cf8cc5a885..96d86b7278ff19a056582f6ddd89a5b8bfdda48f 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
@@ -39,21 +39,21 @@
}
}
}, {
- "taskId" : 5,
- "index" : 5,
+ "taskId" : 1,
+ "index" : 1,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.505GMT",
+ "launchTime" : "2015-05-06T13:03:06.502GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 30,
+ "executorDeserializeTime" : 31,
"executorRunTime" : 350,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 1,
+ "resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -74,26 +74,26 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 3675510,
+ "writeTime" : 3934399,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 1,
- "index" : 1,
+ "taskId" : 5,
+ "index" : 5,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.502GMT",
+ "launchTime" : "2015-05-06T13:03:06.505GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 31,
+ "executorDeserializeTime" : 30,
"executorRunTime" : 350,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 0,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -114,22 +114,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 3934399,
+ "writeTime" : 3675510,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 4,
- "index" : 4,
+ "taskId" : 0,
+ "index" : 0,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.504GMT",
+ "launchTime" : "2015-05-06T13:03:06.494GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 31,
+ "executorDeserializeTime" : 32,
"executorRunTime" : 349,
"resultSize" : 2010,
"jvmGcTime" : 7,
@@ -137,7 +137,7 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 60488,
+ "bytesRead" : 49294,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -154,15 +154,15 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 83022,
+ "writeTime" : 3842811,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 7,
- "index" : 7,
+ "taskId" : 3,
+ "index" : 3,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.506GMT",
+ "launchTime" : "2015-05-06T13:03:06.504GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -173,7 +173,7 @@
"executorRunTime" : 349,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 0,
+ "resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -194,13 +194,13 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 2579051,
+ "writeTime" : 1311694,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 3,
- "index" : 3,
+ "taskId" : 4,
+ "index" : 4,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.504GMT",
"executorId" : "driver",
@@ -213,7 +213,7 @@
"executorRunTime" : 349,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 2,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -234,30 +234,30 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 1311694,
+ "writeTime" : 83022,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 0,
- "index" : 0,
+ "taskId" : 7,
+ "index" : 7,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.494GMT",
+ "launchTime" : "2015-05-06T13:03:06.506GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 32,
+ "executorDeserializeTime" : 31,
"executorRunTime" : 349,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 1,
+ "resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 49294,
+ "bytesRead" : 60488,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -274,7 +274,7 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 3842811,
+ "writeTime" : 2579051,
"recordsWritten" : 10
}
}
@@ -479,25 +479,25 @@
}
}
}, {
- "taskId" : 16,
- "index" : 16,
+ "taskId" : 9,
+ "index" : 9,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.001GMT",
+ "launchTime" : "2015-05-06T13:03:06.915GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 10,
+ "executorDeserializeTime" : 9,
"executorRunTime" : 84,
"resultSize" : 2010,
- "jvmGcTime" : 5,
+ "jvmGcTime" : 0,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 70564,
+ "bytesRead" : 60489,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -514,22 +514,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 108320,
+ "writeTime" : 101664,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 19,
- "index" : 19,
+ "taskId" : 16,
+ "index" : 16,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.012GMT",
+ "launchTime" : "2015-05-06T13:03:07.001GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 5,
+ "executorDeserializeTime" : 10,
"executorRunTime" : 84,
"resultSize" : 2010,
"jvmGcTime" : 5,
@@ -554,30 +554,30 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 95788,
+ "writeTime" : 108320,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 9,
- "index" : 9,
+ "taskId" : 19,
+ "index" : 19,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.915GMT",
+ "launchTime" : "2015-05-06T13:03:07.012GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 9,
+ "executorDeserializeTime" : 5,
"executorRunTime" : 84,
"resultSize" : 2010,
- "jvmGcTime" : 0,
+ "jvmGcTime" : 5,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 60489,
+ "bytesRead" : 70564,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -594,25 +594,25 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 101664,
+ "writeTime" : 95788,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 20,
- "index" : 20,
+ "taskId" : 14,
+ "index" : 14,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.014GMT",
+ "launchTime" : "2015-05-06T13:03:06.925GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 3,
+ "executorDeserializeTime" : 6,
"executorRunTime" : 83,
"resultSize" : 2010,
- "jvmGcTime" : 5,
+ "jvmGcTime" : 0,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
@@ -634,25 +634,25 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 97716,
+ "writeTime" : 95646,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 14,
- "index" : 14,
+ "taskId" : 20,
+ "index" : 20,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.925GMT",
+ "launchTime" : "2015-05-06T13:03:07.014GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 6,
+ "executorDeserializeTime" : 3,
"executorRunTime" : 83,
"resultSize" : 2010,
- "jvmGcTime" : 0,
+ "jvmGcTime" : 5,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
@@ -674,7 +674,7 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 95646,
+ "writeTime" : 97716,
"recordsWritten" : 10
}
}
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
index 11eec0b49c40b16f5024d2540d0886cf8cc5a885..96d86b7278ff19a056582f6ddd89a5b8bfdda48f 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@@ -39,21 +39,21 @@
}
}
}, {
- "taskId" : 5,
- "index" : 5,
+ "taskId" : 1,
+ "index" : 1,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.505GMT",
+ "launchTime" : "2015-05-06T13:03:06.502GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 30,
+ "executorDeserializeTime" : 31,
"executorRunTime" : 350,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 1,
+ "resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -74,26 +74,26 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 3675510,
+ "writeTime" : 3934399,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 1,
- "index" : 1,
+ "taskId" : 5,
+ "index" : 5,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.502GMT",
+ "launchTime" : "2015-05-06T13:03:06.505GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 31,
+ "executorDeserializeTime" : 30,
"executorRunTime" : 350,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 0,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -114,22 +114,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 3934399,
+ "writeTime" : 3675510,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 4,
- "index" : 4,
+ "taskId" : 0,
+ "index" : 0,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.504GMT",
+ "launchTime" : "2015-05-06T13:03:06.494GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 31,
+ "executorDeserializeTime" : 32,
"executorRunTime" : 349,
"resultSize" : 2010,
"jvmGcTime" : 7,
@@ -137,7 +137,7 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 60488,
+ "bytesRead" : 49294,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -154,15 +154,15 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 83022,
+ "writeTime" : 3842811,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 7,
- "index" : 7,
+ "taskId" : 3,
+ "index" : 3,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.506GMT",
+ "launchTime" : "2015-05-06T13:03:06.504GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -173,7 +173,7 @@
"executorRunTime" : 349,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 0,
+ "resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -194,13 +194,13 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 2579051,
+ "writeTime" : 1311694,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 3,
- "index" : 3,
+ "taskId" : 4,
+ "index" : 4,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.504GMT",
"executorId" : "driver",
@@ -213,7 +213,7 @@
"executorRunTime" : 349,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 2,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -234,30 +234,30 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 1311694,
+ "writeTime" : 83022,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 0,
- "index" : 0,
+ "taskId" : 7,
+ "index" : 7,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.494GMT",
+ "launchTime" : "2015-05-06T13:03:06.506GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 32,
+ "executorDeserializeTime" : 31,
"executorRunTime" : 349,
"resultSize" : 2010,
"jvmGcTime" : 7,
- "resultSerializationTime" : 1,
+ "resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 49294,
+ "bytesRead" : 60488,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -274,7 +274,7 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 3842811,
+ "writeTime" : 2579051,
"recordsWritten" : 10
}
}
@@ -479,25 +479,25 @@
}
}
}, {
- "taskId" : 16,
- "index" : 16,
+ "taskId" : 9,
+ "index" : 9,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.001GMT",
+ "launchTime" : "2015-05-06T13:03:06.915GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 10,
+ "executorDeserializeTime" : 9,
"executorRunTime" : 84,
"resultSize" : 2010,
- "jvmGcTime" : 5,
+ "jvmGcTime" : 0,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 70564,
+ "bytesRead" : 60489,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -514,22 +514,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 108320,
+ "writeTime" : 101664,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 19,
- "index" : 19,
+ "taskId" : 16,
+ "index" : 16,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.012GMT",
+ "launchTime" : "2015-05-06T13:03:07.001GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 5,
+ "executorDeserializeTime" : 10,
"executorRunTime" : 84,
"resultSize" : 2010,
"jvmGcTime" : 5,
@@ -554,30 +554,30 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 95788,
+ "writeTime" : 108320,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 9,
- "index" : 9,
+ "taskId" : 19,
+ "index" : 19,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.915GMT",
+ "launchTime" : "2015-05-06T13:03:07.012GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 9,
+ "executorDeserializeTime" : 5,
"executorRunTime" : 84,
"resultSize" : 2010,
- "jvmGcTime" : 0,
+ "jvmGcTime" : 5,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 60489,
+ "bytesRead" : 70564,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -594,25 +594,25 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 101664,
+ "writeTime" : 95788,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 20,
- "index" : 20,
+ "taskId" : 14,
+ "index" : 14,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.014GMT",
+ "launchTime" : "2015-05-06T13:03:06.925GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 3,
+ "executorDeserializeTime" : 6,
"executorRunTime" : 83,
"resultSize" : 2010,
- "jvmGcTime" : 5,
+ "jvmGcTime" : 0,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
@@ -634,25 +634,25 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 97716,
+ "writeTime" : 95646,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 14,
- "index" : 14,
+ "taskId" : 20,
+ "index" : 20,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:06.925GMT",
+ "launchTime" : "2015-05-06T13:03:07.014GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 6,
+ "executorDeserializeTime" : 3,
"executorRunTime" : 83,
"resultSize" : 2010,
- "jvmGcTime" : 0,
+ "jvmGcTime" : 5,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
@@ -674,7 +674,7 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 95646,
+ "writeTime" : 97716,
"recordsWritten" : 10
}
}
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
index 9528d872ef731083eb1cdc0ccf2675315739da14..e0e9e8140c7175efafe7a146f0dd0d948a3910f9 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
@@ -39,21 +39,21 @@
}
}
}, {
- "taskId" : 86,
- "index" : 86,
+ "taskId" : 41,
+ "index" : 41,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.374GMT",
+ "launchTime" : "2015-05-06T13:03:07.200GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 3,
+ "executorDeserializeTime" : 2,
"executorRunTime" : 16,
"resultSize" : 2065,
"jvmGcTime" : 0,
- "resultSerializationTime" : 1,
+ "resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -74,15 +74,15 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 95848,
+ "writeTime" : 90765,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 41,
- "index" : 41,
+ "taskId" : 43,
+ "index" : 43,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.200GMT",
+ "launchTime" : "2015-05-06T13:03:07.204GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -114,22 +114,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 90765,
+ "writeTime" : 171516,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 68,
- "index" : 68,
+ "taskId" : 57,
+ "index" : 57,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.306GMT",
+ "launchTime" : "2015-05-06T13:03:07.257GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 2,
+ "executorDeserializeTime" : 3,
"executorRunTime" : 16,
"resultSize" : 2065,
"jvmGcTime" : 0,
@@ -154,7 +154,7 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 101750,
+ "writeTime" : 96849,
"recordsWritten" : 10
}
}
@@ -199,10 +199,10 @@
}
}
}, {
- "taskId" : 43,
- "index" : 43,
+ "taskId" : 68,
+ "index" : 68,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.204GMT",
+ "launchTime" : "2015-05-06T13:03:07.306GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -234,15 +234,15 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 171516,
+ "writeTime" : 101750,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 57,
- "index" : 57,
+ "taskId" : 86,
+ "index" : 86,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.257GMT",
+ "launchTime" : "2015-05-06T13:03:07.374GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -253,7 +253,7 @@
"executorRunTime" : 16,
"resultSize" : 2065,
"jvmGcTime" : 0,
- "resultSerializationTime" : 0,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -274,15 +274,15 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 96849,
+ "writeTime" : 95848,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 59,
- "index" : 59,
+ "taskId" : 32,
+ "index" : 32,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.265GMT",
+ "launchTime" : "2015-05-06T13:03:07.148GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -314,22 +314,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 100753,
+ "writeTime" : 89603,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 32,
- "index" : 32,
+ "taskId" : 39,
+ "index" : 39,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.148GMT",
+ "launchTime" : "2015-05-06T13:03:07.180GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 3,
+ "executorDeserializeTime" : 2,
"executorRunTime" : 17,
"resultSize" : 2065,
"jvmGcTime" : 0,
@@ -354,22 +354,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 89603,
+ "writeTime" : 98748,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 87,
- "index" : 87,
+ "taskId" : 42,
+ "index" : 42,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.374GMT",
+ "launchTime" : "2015-05-06T13:03:07.203GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
+ "executorDeserializeTime" : 10,
"executorRunTime" : 17,
"resultSize" : 2065,
"jvmGcTime" : 0,
@@ -394,15 +394,15 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 102159,
+ "writeTime" : 103713,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 99,
- "index" : 99,
+ "taskId" : 51,
+ "index" : 51,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.426GMT",
+ "launchTime" : "2015-05-06T13:03:07.242GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -417,7 +417,7 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 70565,
+ "bytesRead" : 70564,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -434,25 +434,25 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 133964,
+ "writeTime" : 96013,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 63,
- "index" : 63,
+ "taskId" : 59,
+ "index" : 59,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.276GMT",
+ "launchTime" : "2015-05-06T13:03:07.265GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 20,
+ "executorDeserializeTime" : 3,
"executorRunTime" : 17,
"resultSize" : 2065,
- "jvmGcTime" : 5,
+ "jvmGcTime" : 0,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
@@ -474,25 +474,25 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 102779,
+ "writeTime" : 100753,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 90,
- "index" : 90,
+ "taskId" : 63,
+ "index" : 63,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.385GMT",
+ "launchTime" : "2015-05-06T13:03:07.276GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 2,
+ "executorDeserializeTime" : 20,
"executorRunTime" : 17,
"resultSize" : 2065,
- "jvmGcTime" : 0,
+ "jvmGcTime" : 5,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
@@ -514,22 +514,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 98472,
+ "writeTime" : 102779,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 39,
- "index" : 39,
+ "taskId" : 87,
+ "index" : 87,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.180GMT",
+ "launchTime" : "2015-05-06T13:03:07.374GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 2,
+ "executorDeserializeTime" : 12,
"executorRunTime" : 17,
"resultSize" : 2065,
"jvmGcTime" : 0,
@@ -554,22 +554,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 98748,
+ "writeTime" : 102159,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 42,
- "index" : 42,
+ "taskId" : 90,
+ "index" : 90,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.203GMT",
+ "launchTime" : "2015-05-06T13:03:07.385GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 10,
+ "executorDeserializeTime" : 2,
"executorRunTime" : 17,
"resultSize" : 2065,
"jvmGcTime" : 0,
@@ -594,15 +594,15 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 103713,
+ "writeTime" : 98472,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 51,
- "index" : 51,
+ "taskId" : 99,
+ "index" : 99,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.242GMT",
+ "launchTime" : "2015-05-06T13:03:07.426GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -617,7 +617,7 @@
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
- "bytesRead" : 70564,
+ "bytesRead" : 70565,
"recordsRead" : 10000
},
"outputMetrics" : {
@@ -634,22 +634,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 96013,
+ "writeTime" : 133964,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 50,
- "index" : 50,
+ "taskId" : 44,
+ "index" : 44,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.240GMT",
+ "launchTime" : "2015-05-06T13:03:07.205GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 4,
+ "executorDeserializeTime" : 3,
"executorRunTime" : 18,
"resultSize" : 2065,
"jvmGcTime" : 0,
@@ -674,22 +674,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 90836,
+ "writeTime" : 98293,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 53,
- "index" : 53,
+ "taskId" : 47,
+ "index" : 47,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.244GMT",
+ "launchTime" : "2015-05-06T13:03:07.212GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 6,
+ "executorDeserializeTime" : 2,
"executorRunTime" : 18,
"resultSize" : 2065,
"jvmGcTime" : 0,
@@ -714,22 +714,22 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 92835,
+ "writeTime" : 103015,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 44,
- "index" : 44,
+ "taskId" : 50,
+ "index" : 50,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.205GMT",
+ "launchTime" : "2015-05-06T13:03:07.240GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 3,
+ "executorDeserializeTime" : 4,
"executorRunTime" : 18,
"resultSize" : 2065,
"jvmGcTime" : 0,
@@ -754,25 +754,25 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 98293,
+ "writeTime" : 90836,
"recordsWritten" : 10
}
}
}, {
- "taskId" : 80,
- "index" : 80,
+ "taskId" : 52,
+ "index" : 52,
"attempt" : 0,
- "launchTime" : "2015-05-06T13:03:07.341GMT",
+ "launchTime" : "2015-05-06T13:03:07.243GMT",
"executorId" : "driver",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 13,
+ "executorDeserializeTime" : 5,
"executorRunTime" : 18,
"resultSize" : 2065,
- "jvmGcTime" : 5,
+ "jvmGcTime" : 0,
"resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
@@ -794,7 +794,7 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
- "writeTime" : 98069,
+ "writeTime" : 89664,
"recordsWritten" : 10
}
}
diff --git a/core/src/test/scala/org/apache/spark/status/api/v1/AllStagesResourceSuite.scala b/core/src/test/scala/org/apache/spark/status/api/v1/AllStagesResourceSuite.scala
index f684e16c25f7c5e455d1cf57ce69911883ea2b9d..1bfb0c1547ec4eca6c2c46ff3050df482387ffb3 100644
--- a/core/src/test/scala/org/apache/spark/status/api/v1/AllStagesResourceSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/api/v1/AllStagesResourceSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.status.api.v1
import java.util.Date
-import scala.collection.mutable.HashMap
+import scala.collection.mutable.LinkedHashMap
import org.apache.spark.SparkFunSuite
import org.apache.spark.scheduler.{StageInfo, TaskInfo, TaskLocality}
@@ -28,7 +28,7 @@ import org.apache.spark.ui.jobs.UIData.{StageUIData, TaskUIData}
class AllStagesResourceSuite extends SparkFunSuite {
def getFirstTaskLaunchTime(taskLaunchTimes: Seq[Long]): Option[Date] = {
- val tasks = new HashMap[Long, TaskUIData]
+ val tasks = new LinkedHashMap[Long, TaskUIData]
taskLaunchTimes.zipWithIndex.foreach { case (time, idx) =>
tasks(idx.toLong) = TaskUIData(
new TaskInfo(idx, idx, 1, time, "", "", TaskLocality.ANY, false), None)
diff --git a/docs/configuration.md b/docs/configuration.md
index 4bda464b98bf64b015118ea1394178f9e909f706..2f801961050e1c9274f3a81aee6d7846dca2a97a 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -626,6 +626,14 @@ Apart from these, the following properties are also available, and may be useful
collecting.
</td>
</tr>
+<tr>
+ <td><code>spark.ui.retainedTasks</code></td>
+ <td>100000</td>
+ <td>
+ How many tasks the Spark UI and status APIs remember before garbage
+ collecting.
+ </td>
+</tr>
<tr>
<td><code>spark.worker.ui.retainedExecutors</code></td>
<td>1000</td>