Skip to content
Snippets Groups Projects
Commit 368b8c2b authored by Hari Shreedharan's avatar Hari Shreedharan Committed by Andrew Or
Browse files

[HOTFIX] Add tests for SparkListenerApplicationStart with Driver Logs.

#6166 added the driver logs to `SparkListenerApplicationStart`. This  adds tests in `JsonProtocolSuite` to ensure we don't regress.

Author: Hari Shreedharan <hshreedharan@apache.org>

Closes #6368 from harishreedharan/jsonprotocol-test and squashes the following commits:

dc9eafc [Hari Shreedharan] [HOTFIX] Add tests for SparkListenerApplicationStart with Driver Logs.
parent baa89838
No related branches found
No related tags found
No related merge requests found
......@@ -75,10 +75,12 @@ class JsonProtocolSuite extends FunSuite {
val blockManagerRemoved = SparkListenerBlockManagerRemoved(2L,
BlockManagerId("Scarce", "to be counted...", 100))
val unpersistRdd = SparkListenerUnpersistRDD(12345)
val logUrlMap = Map("stderr" -> "mystderr", "stdout" -> "mystdout").toMap
val applicationStart = SparkListenerApplicationStart("The winner of all", Some("appId"),
42L, "Garfield", Some("appAttempt"))
val applicationStartWithLogs = SparkListenerApplicationStart("The winner of all", Some("appId"),
42L, "Garfield", Some("appAttempt"), Some(logUrlMap))
val applicationEnd = SparkListenerApplicationEnd(42L)
val logUrlMap = Map("stderr" -> "mystderr", "stdout" -> "mystdout").toMap
val executorAdded = SparkListenerExecutorAdded(executorAddedTime, "exec1",
new ExecutorInfo("Hostee.awesome.com", 11, logUrlMap))
val executorRemoved = SparkListenerExecutorRemoved(executorRemovedTime, "exec2", "test reason")
......@@ -97,6 +99,7 @@ class JsonProtocolSuite extends FunSuite {
testEvent(blockManagerRemoved, blockManagerRemovedJsonString)
testEvent(unpersistRdd, unpersistRDDJsonString)
testEvent(applicationStart, applicationStartJsonString)
testEvent(applicationStartWithLogs, applicationStartJsonWithLogUrlsString)
testEvent(applicationEnd, applicationEndJsonString)
testEvent(executorAdded, executorAddedJsonString)
testEvent(executorRemoved, executorRemovedJsonString)
......@@ -277,10 +280,12 @@ class JsonProtocolSuite extends FunSuite {
test("SparkListenerApplicationStart backwards compatibility") {
// SparkListenerApplicationStart in Spark 1.0.0 do not have an "appId" property.
// SparkListenerApplicationStart pre-Spark 1.4 does not have "appAttemptId".
val applicationStart = SparkListenerApplicationStart("test", None, 1L, "user", None)
// SparkListenerApplicationStart pre-Spark 1.5 does not have "driverLogs
val applicationStart = SparkListenerApplicationStart("test", None, 1L, "user", None, None)
val oldEvent = JsonProtocol.applicationStartToJson(applicationStart)
.removeField({ _._1 == "App ID" })
.removeField({ _._1 == "App Attempt ID" })
.removeField({ _._1 == "Driver Logs"})
assert(applicationStart === JsonProtocol.applicationStartFromJson(oldEvent))
}
......@@ -1544,6 +1549,22 @@ class JsonProtocolSuite extends FunSuite {
|}
"""
private val applicationStartJsonWithLogUrlsString =
"""
|{
| "Event": "SparkListenerApplicationStart",
| "App Name": "The winner of all",
| "App ID": "appId",
| "Timestamp": 42,
| "User": "Garfield",
| "App Attempt ID": "appAttempt",
| "Driver Logs" : {
| "stderr" : "mystderr",
| "stdout" : "mystdout"
| }
|}
"""
private val applicationEndJsonString =
"""
|{
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment