Skip to content
Snippets Groups Projects
Commit bb76eae1 authored by AbhishekKr's avatar AbhishekKr Committed by Reynold Xin
Browse files

[python alternative] pyspark require Python2, failing if system default is Py3 from shell.py

Python alternative for https://github.com/apache/spark/pull/392; managed from shell.py

Author: AbhishekKr <abhikumar163@gmail.com>

Closes #399 from abhishekkr/pyspark_shell and squashes the following commits:

134bdc9 [AbhishekKr] pyspark require Python2, failing if system default is Py3 from shell.py
parent 6ad4c549
No related branches found
No related tags found
No related merge requests found
...@@ -20,6 +20,14 @@ An interactive shell. ...@@ -20,6 +20,14 @@ An interactive shell.
This file is designed to be launched as a PYTHONSTARTUP script. This file is designed to be launched as a PYTHONSTARTUP script.
""" """
import sys
if sys.version_info.major != 2:
print("Error: Default Python used is Python%s" % sys.version_info.major)
print("\tSet env variable PYSPARK_PYTHON to Python2 binary and re-run it.")
sys.exit(1)
import os import os
import platform import platform
import pyspark import pyspark
...@@ -34,21 +42,21 @@ if os.environ.get("SPARK_EXECUTOR_URI"): ...@@ -34,21 +42,21 @@ if os.environ.get("SPARK_EXECUTOR_URI"):
sc = SparkContext(os.environ.get("MASTER", "local[*]"), "PySparkShell", pyFiles=add_files) sc = SparkContext(os.environ.get("MASTER", "local[*]"), "PySparkShell", pyFiles=add_files)
print """Welcome to print("""Welcome to
____ __ ____ __
/ __/__ ___ _____/ /__ / __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/ _\ \/ _ \/ _ `/ __/ '_/
/__ / .__/\_,_/_/ /_/\_\ version 1.0.0-SNAPSHOT /__ / .__/\_,_/_/ /_/\_\ version 1.0.0-SNAPSHOT
/_/ /_/
""" """)
print "Using Python version %s (%s, %s)" % ( print("Using Python version %s (%s, %s)" % (
platform.python_version(), platform.python_version(),
platform.python_build()[0], platform.python_build()[0],
platform.python_build()[1]) platform.python_build()[1]))
print "Spark context available as sc." print("Spark context available as sc.")
if add_files != None: if add_files != None:
print "Adding files: [%s]" % ", ".join(add_files) print("Adding files: [%s]" % ", ".join(add_files))
# The ./bin/pyspark script stores the old PYTHONSTARTUP value in OLD_PYTHONSTARTUP, # The ./bin/pyspark script stores the old PYTHONSTARTUP value in OLD_PYTHONSTARTUP,
# which allows us to execute the user's PYTHONSTARTUP file: # which allows us to execute the user's PYTHONSTARTUP file:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment