From b58340dbd9a741331fc4c3829b08c093560056c2 Mon Sep 17 00:00:00 2001
From: Josh Rosen <joshrosen@eecs.berkeley.edu>
Date: Tue, 1 Jan 2013 14:48:45 -0800
Subject: [PATCH] Rename top-level 'pyspark' directory to 'python'
---
.../src/main/scala/spark/api/python/PythonRDD.scala | 2 +-
docs/_plugins/copy_api_dirs.rb | 8 ++++----
pyspark-shell | 3 +++
pyspark/pyspark-shell | 3 ---
{pyspark => python}/.gitignore | 0
{pyspark => python}/epydoc.conf | 0
{pyspark => python}/examples/kmeans.py | 0
{pyspark => python}/examples/logistic_regression.py | 0
{pyspark => python}/examples/pi.py | 0
{pyspark => python}/examples/transitive_closure.py | 0
{pyspark => python}/examples/wordcount.py | 0
{pyspark => python}/lib/PY4J_LICENSE.txt | 0
{pyspark => python}/lib/PY4J_VERSION.txt | 0
{pyspark => python}/lib/py4j0.7.egg | Bin
{pyspark => python}/lib/py4j0.7.jar | Bin
{pyspark => python}/pyspark/__init__.py | 2 +-
{pyspark => python}/pyspark/broadcast.py | 0
{pyspark => python}/pyspark/cloudpickle.py | 0
{pyspark => python}/pyspark/context.py | 0
{pyspark => python}/pyspark/java_gateway.py | 0
{pyspark => python}/pyspark/join.py | 0
{pyspark => python}/pyspark/rdd.py | 0
{pyspark => python}/pyspark/serializers.py | 0
{pyspark => python}/pyspark/shell.py | 0
{pyspark => python}/pyspark/worker.py | 0
run | 2 +-
pyspark/run-pyspark => run-pyspark | 4 ++--
run2.cmd | 2 +-
28 files changed, 13 insertions(+), 13 deletions(-)
create mode 100755 pyspark-shell
delete mode 100755 pyspark/pyspark-shell
rename {pyspark => python}/.gitignore (100%)
rename {pyspark => python}/epydoc.conf (100%)
rename {pyspark => python}/examples/kmeans.py (100%)
rename {pyspark => python}/examples/logistic_regression.py (100%)
rename {pyspark => python}/examples/pi.py (100%)
rename {pyspark => python}/examples/transitive_closure.py (100%)
rename {pyspark => python}/examples/wordcount.py (100%)
rename {pyspark => python}/lib/PY4J_LICENSE.txt (100%)
rename {pyspark => python}/lib/PY4J_VERSION.txt (100%)
rename {pyspark => python}/lib/py4j0.7.egg (100%)
rename {pyspark => python}/lib/py4j0.7.jar (100%)
rename {pyspark => python}/pyspark/__init__.py (82%)
rename {pyspark => python}/pyspark/broadcast.py (100%)
rename {pyspark => python}/pyspark/cloudpickle.py (100%)
rename {pyspark => python}/pyspark/context.py (100%)
rename {pyspark => python}/pyspark/java_gateway.py (100%)
rename {pyspark => python}/pyspark/join.py (100%)
rename {pyspark => python}/pyspark/rdd.py (100%)
rename {pyspark => python}/pyspark/serializers.py (100%)
rename {pyspark => python}/pyspark/shell.py (100%)
rename {pyspark => python}/pyspark/worker.py (100%)
rename pyspark/run-pyspark => run-pyspark (86%)
diff --git a/core/src/main/scala/spark/api/python/PythonRDD.scala b/core/src/main/scala/spark/api/python/PythonRDD.scala
index 19a039e330..cf60d14f03 100644
--- a/core/src/main/scala/spark/api/python/PythonRDD.scala
+++ b/core/src/main/scala/spark/api/python/PythonRDD.scala
@@ -38,7 +38,7 @@ private[spark] class PythonRDD[T: ClassManifest](
override def compute(split: Split, context: TaskContext): Iterator[Array[Byte]] = {
val SPARK_HOME = new ProcessBuilder().environment().get("SPARK_HOME")
- val pb = new ProcessBuilder(Seq(pythonExec, SPARK_HOME + "/pyspark/pyspark/worker.py"))
+ val pb = new ProcessBuilder(Seq(pythonExec, SPARK_HOME + "/python/pyspark/worker.py"))
// Add the environmental variables to the process.
val currentEnvVars = pb.environment()
diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
index 577f3ebe70..c9ce589c1b 100644
--- a/docs/_plugins/copy_api_dirs.rb
+++ b/docs/_plugins/copy_api_dirs.rb
@@ -30,8 +30,8 @@ if ENV['SKIP_SCALADOC'] != '1'
end
if ENV['SKIP_EPYDOC'] != '1'
- puts "Moving to pyspark directory and building epydoc."
- cd("../pyspark")
+ puts "Moving to python directory and building epydoc."
+ cd("../python")
puts `epydoc --config epydoc.conf`
puts "Moving back into docs dir."
@@ -40,8 +40,8 @@ if ENV['SKIP_EPYDOC'] != '1'
puts "echo making directory pyspark"
mkdir_p "pyspark"
- puts "cp -r ../pyspark/docs/. api/pyspark"
- cp_r("../pyspark/docs/.", "api/pyspark")
+ puts "cp -r ../python/docs/. api/pyspark"
+ cp_r("../python/docs/.", "api/pyspark")
cd("..")
end
diff --git a/pyspark-shell b/pyspark-shell
new file mode 100755
index 0000000000..27aaac3a26
--- /dev/null
+++ b/pyspark-shell
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+FWDIR="`dirname $0`"
+exec $FWDIR/run-pyspark $FWDIR/python/pyspark/shell.py "$@"
diff --git a/pyspark/pyspark-shell b/pyspark/pyspark-shell
deleted file mode 100755
index e3736826e8..0000000000
--- a/pyspark/pyspark-shell
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-FWDIR="`dirname $0`"
-exec $FWDIR/run-pyspark $FWDIR/pyspark/shell.py "$@"
diff --git a/pyspark/.gitignore b/python/.gitignore
similarity index 100%
rename from pyspark/.gitignore
rename to python/.gitignore
diff --git a/pyspark/epydoc.conf b/python/epydoc.conf
similarity index 100%
rename from pyspark/epydoc.conf
rename to python/epydoc.conf
diff --git a/pyspark/examples/kmeans.py b/python/examples/kmeans.py
similarity index 100%
rename from pyspark/examples/kmeans.py
rename to python/examples/kmeans.py
diff --git a/pyspark/examples/logistic_regression.py b/python/examples/logistic_regression.py
similarity index 100%
rename from pyspark/examples/logistic_regression.py
rename to python/examples/logistic_regression.py
diff --git a/pyspark/examples/pi.py b/python/examples/pi.py
similarity index 100%
rename from pyspark/examples/pi.py
rename to python/examples/pi.py
diff --git a/pyspark/examples/transitive_closure.py b/python/examples/transitive_closure.py
similarity index 100%
rename from pyspark/examples/transitive_closure.py
rename to python/examples/transitive_closure.py
diff --git a/pyspark/examples/wordcount.py b/python/examples/wordcount.py
similarity index 100%
rename from pyspark/examples/wordcount.py
rename to python/examples/wordcount.py
diff --git a/pyspark/lib/PY4J_LICENSE.txt b/python/lib/PY4J_LICENSE.txt
similarity index 100%
rename from pyspark/lib/PY4J_LICENSE.txt
rename to python/lib/PY4J_LICENSE.txt
diff --git a/pyspark/lib/PY4J_VERSION.txt b/python/lib/PY4J_VERSION.txt
similarity index 100%
rename from pyspark/lib/PY4J_VERSION.txt
rename to python/lib/PY4J_VERSION.txt
diff --git a/pyspark/lib/py4j0.7.egg b/python/lib/py4j0.7.egg
similarity index 100%
rename from pyspark/lib/py4j0.7.egg
rename to python/lib/py4j0.7.egg
diff --git a/pyspark/lib/py4j0.7.jar b/python/lib/py4j0.7.jar
similarity index 100%
rename from pyspark/lib/py4j0.7.jar
rename to python/lib/py4j0.7.jar
diff --git a/pyspark/pyspark/__init__.py b/python/pyspark/__init__.py
similarity index 82%
rename from pyspark/pyspark/__init__.py
rename to python/pyspark/__init__.py
index 1ab360a666..c595ae0842 100644
--- a/pyspark/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -10,7 +10,7 @@ Public classes:
"""
import sys
import os
-sys.path.insert(0, os.path.join(os.environ["SPARK_HOME"], "pyspark/lib/py4j0.7.egg"))
+sys.path.insert(0, os.path.join(os.environ["SPARK_HOME"], "python/lib/py4j0.7.egg"))
from pyspark.context import SparkContext
diff --git a/pyspark/pyspark/broadcast.py b/python/pyspark/broadcast.py
similarity index 100%
rename from pyspark/pyspark/broadcast.py
rename to python/pyspark/broadcast.py
diff --git a/pyspark/pyspark/cloudpickle.py b/python/pyspark/cloudpickle.py
similarity index 100%
rename from pyspark/pyspark/cloudpickle.py
rename to python/pyspark/cloudpickle.py
diff --git a/pyspark/pyspark/context.py b/python/pyspark/context.py
similarity index 100%
rename from pyspark/pyspark/context.py
rename to python/pyspark/context.py
diff --git a/pyspark/pyspark/java_gateway.py b/python/pyspark/java_gateway.py
similarity index 100%
rename from pyspark/pyspark/java_gateway.py
rename to python/pyspark/java_gateway.py
diff --git a/pyspark/pyspark/join.py b/python/pyspark/join.py
similarity index 100%
rename from pyspark/pyspark/join.py
rename to python/pyspark/join.py
diff --git a/pyspark/pyspark/rdd.py b/python/pyspark/rdd.py
similarity index 100%
rename from pyspark/pyspark/rdd.py
rename to python/pyspark/rdd.py
diff --git a/pyspark/pyspark/serializers.py b/python/pyspark/serializers.py
similarity index 100%
rename from pyspark/pyspark/serializers.py
rename to python/pyspark/serializers.py
diff --git a/pyspark/pyspark/shell.py b/python/pyspark/shell.py
similarity index 100%
rename from pyspark/pyspark/shell.py
rename to python/pyspark/shell.py
diff --git a/pyspark/pyspark/worker.py b/python/pyspark/worker.py
similarity index 100%
rename from pyspark/pyspark/worker.py
rename to python/pyspark/worker.py
diff --git a/run b/run
index ed788c4db3..08e2b2434b 100755
--- a/run
+++ b/run
@@ -63,7 +63,7 @@ CORE_DIR="$FWDIR/core"
REPL_DIR="$FWDIR/repl"
EXAMPLES_DIR="$FWDIR/examples"
BAGEL_DIR="$FWDIR/bagel"
-PYSPARK_DIR="$FWDIR/pyspark"
+PYSPARK_DIR="$FWDIR/python"
# Build up classpath
CLASSPATH="$SPARK_CLASSPATH"
diff --git a/pyspark/run-pyspark b/run-pyspark
similarity index 86%
rename from pyspark/run-pyspark
rename to run-pyspark
index 4d10fbea8b..deb0d708b3 100755
--- a/pyspark/run-pyspark
+++ b/run-pyspark
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
# Figure out where the Scala framework is installed
-FWDIR="$(cd `dirname $0`; cd ../; pwd)"
+FWDIR="$(cd `dirname $0`; pwd)"
# Export this as SPARK_HOME
export SPARK_HOME="$FWDIR"
@@ -18,7 +18,7 @@ fi
export PYSPARK_PYTHON
# Add the PySpark classes to the Python path:
-export PYTHONPATH=$SPARK_HOME/pyspark/:$PYTHONPATH
+export PYTHONPATH=$SPARK_HOME/python/:$PYTHONPATH
# Launch with `scala` by default:
if [[ "$SPARK_LAUNCH_WITH_SCALA" != "0" ]] ; then
diff --git a/run2.cmd b/run2.cmd
index 9c50804e69..83464b1166 100644
--- a/run2.cmd
+++ b/run2.cmd
@@ -34,7 +34,7 @@ set CORE_DIR=%FWDIR%core
set REPL_DIR=%FWDIR%repl
set EXAMPLES_DIR=%FWDIR%examples
set BAGEL_DIR=%FWDIR%bagel
-set PYSPARK_DIR=%FWDIR%pyspark
+set PYSPARK_DIR=%FWDIR%python
rem Build up classpath
set CLASSPATH=%SPARK_CLASSPATH%;%MESOS_CLASSPATH%;%FWDIR%conf;%CORE_DIR%\target\scala-%SCALA_VERSION%\classes
--
GitLab