Skip to content
Snippets Groups Projects
Commit 8fd5c7bc authored by Andre Schumacher's avatar Andre Schumacher
Browse files

Implementing SPARK-865: Add the equivalent of ADD_JARS to PySpark

Now ADD_FILES uses a comma as file name separator.
parent a4611d66
No related branches found
No related tags found
No related merge requests found
......@@ -7,10 +7,15 @@ import os
import pyspark
from pyspark.context import SparkContext
# this is the equivalent of ADD_JARS
add_files = os.environ.get("ADD_FILES").split(',') if os.environ.get("ADD_FILES") != None else None
sc = SparkContext(os.environ.get("MASTER", "local"), "PySparkShell")
sc = SparkContext(os.environ.get("MASTER", "local"), "PySparkShell", pyFiles=add_files)
print "Spark context avaiable as sc."
if add_files != None:
print "Adding files: [%s]" % ", ".join(add_files)
# The ./pyspark script stores the old PYTHONSTARTUP value in OLD_PYTHONSTARTUP,
# which allows us to execute the user's PYTHONSTARTUP file:
_pythonstartup = os.environ.get('OLD_PYTHONSTARTUP')
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment