Skip to content
Snippets Groups Projects
Commit 9f0dc829 authored by Matei Zaharia's avatar Matei Zaharia
Browse files

Fix TaskMetrics not being serializable

parent 04fb81ff
No related branches found
No related tags found
No related merge requests found
package spark.executor package spark.executor
class TaskMetrics{ class TaskMetrics extends Serializable {
/** /**
* Time taken on the executor to deserialize this task * Time taken on the executor to deserialize this task
*/ */
var executorDeserializeTime: Int = _ var executorDeserializeTime: Int = _
/** /**
* Time the executor spends actually running the task (including fetching shuffle data) * Time the executor spends actually running the task (including fetching shuffle data)
*/ */
var executorRunTime:Int = _ var executorRunTime:Int = _
/** /**
* The number of bytes this task transmitted back to the driver as the TaskResult * The number of bytes this task transmitted back to the driver as the TaskResult
*/ */
...@@ -23,49 +25,54 @@ class TaskMetrics{ ...@@ -23,49 +25,54 @@ class TaskMetrics{
* If this task writes to shuffle output, metrics on the written shuffle data will be collected here * If this task writes to shuffle output, metrics on the written shuffle data will be collected here
*/ */
var shuffleWriteMetrics: Option[ShuffleWriteMetrics] = None var shuffleWriteMetrics: Option[ShuffleWriteMetrics] = None
} }
object TaskMetrics { object TaskMetrics {
private[spark] def empty() : TaskMetrics = new TaskMetrics private[spark] def empty(): TaskMetrics = new TaskMetrics
} }
class ShuffleReadMetrics { class ShuffleReadMetrics extends Serializable {
/** /**
* Total number of blocks fetched in a shuffle (remote or local) * Total number of blocks fetched in a shuffle (remote or local)
*/ */
var totalBlocksFetched : Int = _ var totalBlocksFetched : Int = _
/** /**
* Number of remote blocks fetched in a shuffle * Number of remote blocks fetched in a shuffle
*/ */
var remoteBlocksFetched: Int = _ var remoteBlocksFetched: Int = _
/** /**
* Local blocks fetched in a shuffle * Local blocks fetched in a shuffle
*/ */
var localBlocksFetched: Int = _ var localBlocksFetched: Int = _
/** /**
* Total time to read shuffle data * Total time to read shuffle data
*/ */
var shuffleReadMillis: Long = _ var shuffleReadMillis: Long = _
/** /**
* Total time that is spent blocked waiting for shuffle to fetch remote data * Total time that is spent blocked waiting for shuffle to fetch remote data
*/ */
var remoteFetchWaitTime: Long = _ var remoteFetchWaitTime: Long = _
/** /**
* The total amount of time for all the shuffle fetches. This adds up time from overlapping * The total amount of time for all the shuffle fetches. This adds up time from overlapping
* shuffles, so can be longer than task time * shuffles, so can be longer than task time
*/ */
var remoteFetchTime: Long = _ var remoteFetchTime: Long = _
/** /**
* Total number of remote bytes read from a shuffle * Total number of remote bytes read from a shuffle
*/ */
var remoteBytesRead: Long = _ var remoteBytesRead: Long = _
} }
class ShuffleWriteMetrics { class ShuffleWriteMetrics extends Serializable {
/** /**
* Number of bytes written for a shuffle * Number of bytes written for a shuffle
*/ */
var shuffleBytesWritten: Long = _ var shuffleBytesWritten: Long = _
} }
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment