Skip to content
Snippets Groups Projects
Commit ed25105f authored by Matei Zaharia's avatar Matei Zaharia
Browse files

Merge pull request #174 from ahirreddy/master

Write Spark UI url to driver file on HDFS

This makes the SIMR code path simpler
parents 1a4cfbea 0ea1f8b2
No related branches found
No related tags found
No related merge requests found
...@@ -31,10 +31,6 @@ private[spark] class SimrSchedulerBackend( ...@@ -31,10 +31,6 @@ private[spark] class SimrSchedulerBackend(
val tmpPath = new Path(driverFilePath + "_tmp") val tmpPath = new Path(driverFilePath + "_tmp")
val filePath = new Path(driverFilePath) val filePath = new Path(driverFilePath)
val uiFilePath = driverFilePath + "_ui"
val tmpUiPath = new Path(uiFilePath + "_tmp")
val uiPath = new Path(uiFilePath)
val maxCores = System.getProperty("spark.simr.executor.cores", "1").toInt val maxCores = System.getProperty("spark.simr.executor.cores", "1").toInt
override def start() { override def start() {
...@@ -49,23 +45,17 @@ private[spark] class SimrSchedulerBackend( ...@@ -49,23 +45,17 @@ private[spark] class SimrSchedulerBackend(
logInfo("Writing to HDFS file: " + driverFilePath) logInfo("Writing to HDFS file: " + driverFilePath)
logInfo("Writing Akka address: " + driverUrl) logInfo("Writing Akka address: " + driverUrl)
logInfo("Writing to HDFS file: " + uiFilePath)
logInfo("Writing Spark UI Address: " + sc.ui.appUIAddress) logInfo("Writing Spark UI Address: " + sc.ui.appUIAddress)
// Create temporary file to prevent race condition where executors get empty driverUrl file // Create temporary file to prevent race condition where executors get empty driverUrl file
val temp = fs.create(tmpPath, true) val temp = fs.create(tmpPath, true)
temp.writeUTF(driverUrl) temp.writeUTF(driverUrl)
temp.writeInt(maxCores) temp.writeInt(maxCores)
temp.writeUTF(sc.ui.appUIAddress)
temp.close() temp.close()
// "Atomic" rename // "Atomic" rename
fs.rename(tmpPath, filePath) fs.rename(tmpPath, filePath)
// Write Spark UI Address to file
val uiTemp = fs.create(tmpUiPath, true)
uiTemp.writeUTF(sc.ui.appUIAddress)
uiTemp.close()
fs.rename(tmpUiPath, uiPath)
} }
override def stop() { override def stop() {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment