diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index a35e003cdc1ee5130f0e472a520a98499f40b716..da5a502fd5ffc9d7fe2ae52e94e1c09c0ae01354 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -114,6 +114,8 @@ For example: SPARK_YARN_APP_JAR=examples/target/scala-{{site.SCALA_VERSION}}/spark-examples-assembly-{{site.SPARK_VERSION}}.jar \ MASTER=yarn-client ./bin/spark-shell +You can also send extra files to yarn cluster for worker to use by exporting SPARK_YARN_DIST_FILES=file1,file2... etc. + # Building Spark for Hadoop/YARN 2.2.x See [Building Spark with Maven](building-with-maven.html) for instructions on how to build Spark using the Maven process. diff --git a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala index 324ef4616fe268b4005a2ec014fb0a8636d4e2e2..4b1b5da048df4d801dacb24f7df18245e98735ac 100644 --- a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala +++ b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala @@ -39,6 +39,7 @@ private[spark] class YarnClientSchedulerBackend( val defaultWorkerNumber = "1" val userJar = System.getenv("SPARK_YARN_APP_JAR") + val distFiles = System.getenv("SPARK_YARN_DIST_FILES") var workerCores = System.getenv("SPARK_WORKER_CORES") var workerMemory = System.getenv("SPARK_WORKER_MEMORY") var workerNumber = System.getenv("SPARK_WORKER_INSTANCES") @@ -64,7 +65,8 @@ private[spark] class YarnClientSchedulerBackend( "--worker-memory", workerMemory, "--worker-cores", workerCores, "--num-workers", workerNumber, - "--master-class", "org.apache.spark.deploy.yarn.WorkerLauncher" + "--master-class", "org.apache.spark.deploy.yarn.WorkerLauncher", + "--files", distFiles ) val args = new ClientArguments(argsArray, conf)