Skip to content
Snippets Groups Projects
Commit 0ea1f8b2 authored by Ahir Reddy's avatar Ahir Reddy
Browse files

Write Spark UI url to driver file on HDFS

parent 39af914b
No related branches found
No related tags found
No related merge requests found
......@@ -31,10 +31,6 @@ private[spark] class SimrSchedulerBackend(
val tmpPath = new Path(driverFilePath + "_tmp")
val filePath = new Path(driverFilePath)
val uiFilePath = driverFilePath + "_ui"
val tmpUiPath = new Path(uiFilePath + "_tmp")
val uiPath = new Path(uiFilePath)
val maxCores = System.getProperty("spark.simr.executor.cores", "1").toInt
override def start() {
......@@ -49,23 +45,17 @@ private[spark] class SimrSchedulerBackend(
logInfo("Writing to HDFS file: " + driverFilePath)
logInfo("Writing Akka address: " + driverUrl)
logInfo("Writing to HDFS file: " + uiFilePath)
logInfo("Writing Spark UI Address: " + sc.ui.appUIAddress)
// Create temporary file to prevent race condition where executors get empty driverUrl file
val temp = fs.create(tmpPath, true)
temp.writeUTF(driverUrl)
temp.writeInt(maxCores)
temp.writeUTF(sc.ui.appUIAddress)
temp.close()
// "Atomic" rename
fs.rename(tmpPath, filePath)
// Write Spark UI Address to file
val uiTemp = fs.create(tmpUiPath, true)
uiTemp.writeUTF(sc.ui.appUIAddress)
uiTemp.close()
fs.rename(tmpUiPath, uiPath)
}
override def stop() {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment