diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index c58555fc9d2c5a3cafb1ba89be98906bac197dd8..1a2e774738fe74e9f6883c658de7271654226d77 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -61,13 +61,17 @@ sys.path.insert(0, s)
from pyspark.conf import SparkConf
from pyspark.context import SparkContext
-from pyspark.sql import SQLContext
from pyspark.rdd import RDD
-from pyspark.sql import SchemaRDD
-from pyspark.sql import Row
from pyspark.files import SparkFiles
from pyspark.storagelevel import StorageLevel
+from pyspark.accumulators import Accumulator, AccumulatorParam
+from pyspark.broadcast import Broadcast
+from pyspark.serializers import MarshalSerializer, PickleSerializer
+# for back compatibility
+from pyspark.sql import SQLContext, HiveContext, SchemaRDD, Row
-__all__ = ["SparkConf", "SparkContext", "SQLContext", "RDD", "SchemaRDD",
- "SparkFiles", "StorageLevel", "Row"]
+__all__ = [
+ "SparkConf", "SparkContext", "SparkFiles", "RDD", "StorageLevel", "Broadcast",
+ "Accumulator", "AccumulatorParam", "MarshalSerializer", "PickleSerializer",
+]
diff --git a/python/pyspark/accumulators.py b/python/pyspark/accumulators.py
index f133cf6f7befc1b91e9b57c5931848d7f489e368..ccbca67656c8db95503ebbe2072a722b182feeaa 100644
--- a/python/pyspark/accumulators.py
+++ b/python/pyspark/accumulators.py
@@ -94,6 +94,9 @@ from pyspark.cloudpickle import CloudPickler
from pyspark.serializers import read_int, PickleSerializer
+__all__ = ['Accumulator', 'AccumulatorParam']
+
+
pickleSer = PickleSerializer()
# Holds accumulators registered on the current machine, keyed by ID. This is then used to send
diff --git a/python/pyspark/broadcast.py b/python/pyspark/broadcast.py
index 675a2fcd2ff4ecf249cd72e38cc90b20e66286c8..5c7c9cc161dff617bf086c7f43a2ebbee75e067f 100644
--- a/python/pyspark/broadcast.py
+++ b/python/pyspark/broadcast.py
@@ -31,6 +31,10 @@ import os
from pyspark.serializers import CompressedSerializer, PickleSerializer
+
+__all__ = ['Broadcast']
+
+
# Holds broadcasted data received from Java, keyed by its id.
_broadcastRegistry = {}
@@ -59,11 +63,20 @@ class Broadcast(object):
"""
self.bid = bid
if path is None:
- self.value = value
+ self._value = value
self._jbroadcast = java_broadcast
self._pickle_registry = pickle_registry
self.path = path
+ @property
+ def value(self):
+ """ Return the broadcasted value
+ """
+ if not hasattr(self, "_value") and self.path is not None:
+ ser = CompressedSerializer(PickleSerializer())
+ self._value = ser.load_stream(open(self.path)).next()
+ return self._value
+
def unpersist(self, blocking=False):
self._jbroadcast.unpersist(blocking)
os.unlink(self.path)
@@ -72,15 +85,6 @@ class Broadcast(object):
self._pickle_registry.add(self)
return (_from_id, (self.bid, ))
- def __getattr__(self, item):
- if item == 'value' and self.path is not None:
- ser = CompressedSerializer(PickleSerializer())
- value = ser.load_stream(open(self.path)).next()
- self.value = value
- return value
-
- raise AttributeError(item)
-
if __name__ == "__main__":
import doctest
diff --git a/python/pyspark/conf.py b/python/pyspark/conf.py
index fb716f6753a45aa5dca04f070e6f553e19d148f5..b64875a3f495adf0da4f8703b72212b6f468f168 100644
--- a/python/pyspark/conf.py
+++ b/python/pyspark/conf.py
@@ -54,6 +54,8 @@ spark.home=/path
(u'spark.executorEnv.VAR4', u'value4'), (u'spark.home', u'/path')]
"""
+__all__ = ['SparkConf']
+
class SparkConf(object):
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 6e4fdaa6eec9d97c7f5416c7e6ddb08c4b1ea34c..5a30431568b1642b78af1c4eee678af54d770f24 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -37,6 +37,9 @@ from pyspark.rdd import RDD
from py4j.java_collections import ListConverter
+__all__ = ['SparkContext']
+
+
# These are special default configs for PySpark, they will overwrite
# the default ones for Spark if they are not configured by user.
DEFAULT_CONFIGS = {
diff --git a/python/pyspark/files.py b/python/pyspark/files.py
index 331de9a9b2212be42592d0640716a4a2b7a834d3..797573f49dac8a3421f427c75fd404d4c85b52cf 100644
--- a/python/pyspark/files.py
+++ b/python/pyspark/files.py
@@ -18,6 +18,9 @@
import os
+__all__ = ['SparkFiles']
+
+
class SparkFiles(object):
"""
diff --git a/python/pyspark/mllib/classification.py b/python/pyspark/mllib/classification.py
index ffdda7ee19302cc85158cec06f8fd6a284b43fd0..71ab46b61d7fa8dfd5d1787a83d439dd7d70ec90 100644
--- a/python/pyspark/mllib/classification.py
+++ b/python/pyspark/mllib/classification.py
@@ -30,6 +30,10 @@ from pyspark.mllib.regression import LabeledPoint, LinearModel
from math import exp, log
+__all__ = ['LogisticRegressionModel', 'LogisticRegressionWithSGD', 'SVMModel',
+ 'SVMWithSGD', 'NaiveBayesModel', 'NaiveBayes']
+
+
class LogisticRegressionModel(LinearModel):
"""A linear binary classification model derived from logistic regression.
diff --git a/python/pyspark/mllib/clustering.py b/python/pyspark/mllib/clustering.py
index a0630d1d5c58b6e502f8365b3370f9b64d612ba7..f3e952a1d842a659dcbbcd01eb130a4ad46ad3bf 100644
--- a/python/pyspark/mllib/clustering.py
+++ b/python/pyspark/mllib/clustering.py
@@ -25,6 +25,8 @@ from pyspark.mllib._common import \
_get_initial_weights, _serialize_rating, _regression_train_wrapper
from pyspark.mllib.linalg import SparseVector
+__all__ = ['KMeansModel', 'KMeans']
+
class KMeansModel(object):
diff --git a/python/pyspark/mllib/linalg.py b/python/pyspark/mllib/linalg.py
index f485a69db1fa2239c76b9b3943b14da94b883a55..e69051c104e37b61b772184b49d2d9f1a1d7a536 100644
--- a/python/pyspark/mllib/linalg.py
+++ b/python/pyspark/mllib/linalg.py
@@ -27,6 +27,9 @@ import numpy
from numpy import array, array_equal, ndarray, float64, int32
+__all__ = ['SparseVector', 'Vectors']
+
+
class SparseVector(object):
"""
diff --git a/python/pyspark/mllib/random.py b/python/pyspark/mllib/random.py
index 4dc1a4a9124213c92f6f6afa1caf8124ce5df300..3e59c73db85e39a6ee20527ecadf2ab81fcb4ce1 100644
--- a/python/pyspark/mllib/random.py
+++ b/python/pyspark/mllib/random.py
@@ -25,6 +25,9 @@ from pyspark.mllib._common import _deserialize_double, _deserialize_double_vecto
from pyspark.serializers import NoOpSerializer
+__all__ = ['RandomRDDs', ]
+
+
class RandomRDDs:
"""
Generator methods for creating RDDs comprised of i.i.d samples from
diff --git a/python/pyspark/mllib/recommendation.py b/python/pyspark/mllib/recommendation.py
index e863fc249ec3649fcd194c694ba687162f6941b6..2df23394da6f80bde9dc87520b9eec4d99145904 100644
--- a/python/pyspark/mllib/recommendation.py
+++ b/python/pyspark/mllib/recommendation.py
@@ -24,6 +24,8 @@ from pyspark.mllib._common import \
_serialize_tuple, RatingDeserializer
from pyspark.rdd import RDD
+__all__ = ['MatrixFactorizationModel', 'ALS']
+
class MatrixFactorizationModel(object):
diff --git a/python/pyspark/mllib/regression.py b/python/pyspark/mllib/regression.py
index d8792cf44872f005cbcc3b2c09b0541c72843e05..f572dcfb840b6d4b9f5ce3507246e3751b4df7d4 100644
--- a/python/pyspark/mllib/regression.py
+++ b/python/pyspark/mllib/regression.py
@@ -17,15 +17,15 @@
from numpy import array, ndarray
from pyspark import SparkContext
-from pyspark.mllib._common import \
- _dot, _get_unmangled_rdd, _get_unmangled_double_vector_rdd, \
- _serialize_double_matrix, _deserialize_double_matrix, \
- _serialize_double_vector, _deserialize_double_vector, \
- _get_initial_weights, _serialize_rating, _regression_train_wrapper, \
+from pyspark.mllib._common import _dot, _regression_train_wrapper, \
_linear_predictor_typecheck, _have_scipy, _scipy_issparse
from pyspark.mllib.linalg import SparseVector, Vectors
+__all__ = ['LabeledPoint', 'LinearModel', 'LinearRegressionModel', 'RidgeRegressionModel'
+ 'LinearRegressionWithSGD', 'LassoWithSGD', 'RidgeRegressionWithSGD']
+
+
class LabeledPoint(object):
"""
diff --git a/python/pyspark/mllib/stat.py b/python/pyspark/mllib/stat.py
index feef0d16cd644905a48379cf84ebc1d4cb8fa640..8c726f171c978f7682e24a822dedfb6b1e91537f 100644
--- a/python/pyspark/mllib/stat.py
+++ b/python/pyspark/mllib/stat.py
@@ -21,8 +21,10 @@ Python package for statistical functions in MLlib.
from pyspark.mllib._common import \
_get_unmangled_double_vector_rdd, _get_unmangled_rdd, \
- _serialize_double, _serialize_double_vector, \
- _deserialize_double, _deserialize_double_matrix, _deserialize_double_vector
+ _serialize_double, _deserialize_double_matrix, _deserialize_double_vector
+
+
+__all__ = ['MultivariateStatisticalSummary', 'Statistics']
class MultivariateStatisticalSummary(object):
diff --git a/python/pyspark/mllib/tree.py b/python/pyspark/mllib/tree.py
index e9d778df5a24b49b0d3b55b088feb18cc48cf4ef..a2fade61e9a71d41bb9cfafc1a5eae48873cf260 100644
--- a/python/pyspark/mllib/tree.py
+++ b/python/pyspark/mllib/tree.py
@@ -26,6 +26,9 @@ from pyspark.mllib.regression import LabeledPoint
from pyspark.serializers import NoOpSerializer
+__all__ = ['DecisionTreeModel', 'DecisionTree']
+
+
class DecisionTreeModel(object):
"""
@@ -88,6 +91,7 @@ class DecisionTree(object):
It will probably be modified for Spark v1.2.
Example usage:
+
>>> from numpy import array
>>> import sys
>>> from pyspark.mllib.regression import LabeledPoint
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
index 6fc9f66bc5a942bfa7775abc3ba5fdb80c2e6140..dff6fc26fcb18b2d1248693e3de3e8efc214078c 100644
--- a/python/pyspark/rdd.py
+++ b/python/pyspark/rdd.py
@@ -48,6 +48,7 @@ from pyspark.shuffle import Aggregator, InMemoryMerger, ExternalMerger, \
from py4j.java_collections import ListConverter, MapConverter
+
__all__ = ["RDD"]
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py
index fc49aa42dbaf9e6f9a8e50b8bab563c4827ff7be..55e6cf330861154bda4dbb1f6e7361661c460384 100644
--- a/python/pyspark/serializers.py
+++ b/python/pyspark/serializers.py
@@ -409,7 +409,7 @@ class AutoSerializer(FramedSerializer):
class CompressedSerializer(FramedSerializer):
"""
- compress the serialized data
+ Compress the serialized data
"""
def __init__(self, serializer):
diff --git a/python/pyspark/sql.py b/python/pyspark/sql.py
index 0ff6a548a85f176212d73dbcae73206e7a500dc4..44316926ba33479bf43c37e163d037f8464b1a73 100644
--- a/python/pyspark/sql.py
+++ b/python/pyspark/sql.py
@@ -40,8 +40,7 @@ __all__ = [
"StringType", "BinaryType", "BooleanType", "TimestampType", "DecimalType",
"DoubleType", "FloatType", "ByteType", "IntegerType", "LongType",
"ShortType", "ArrayType", "MapType", "StructField", "StructType",
- "SQLContext", "HiveContext", "LocalHiveContext", "TestHiveContext",
- "SchemaRDD", "Row"]
+ "SQLContext", "HiveContext", "SchemaRDD", "Row"]
class DataType(object):
@@ -1037,7 +1036,7 @@ class SQLContext:
"can not infer schema")
if type(first) is dict:
warnings.warn("Using RDD of dict to inferSchema is deprecated,"
- "please use pyspark.Row instead")
+ "please use pyspark.sql.Row instead")
schema = _infer_schema(first)
rdd = rdd.mapPartitions(lambda rows: _drop_schema(rows, schema))
@@ -1487,6 +1486,21 @@ class Row(tuple):
return "<Row(%s)>" % ", ".join(self)
+def inherit_doc(cls):
+ for name, func in vars(cls).items():
+ # only inherit docstring for public functions
+ if name.startswith("_"):
+ continue
+ if not func.__doc__:
+ for parent in cls.__bases__:
+ parent_func = getattr(parent, name, None)
+ if parent_func and getattr(parent_func, "__doc__", None):
+ func.__doc__ = parent_func.__doc__
+ break
+ return cls
+
+
+@inherit_doc
class SchemaRDD(RDD):
"""An RDD of L{Row} objects that has an associated schema.
@@ -1563,6 +1577,7 @@ class SchemaRDD(RDD):
self._jschema_rdd.registerTempTable(name)
def registerAsTable(self, name):
+ """DEPRECATED: use registerTempTable() instead"""
warnings.warn("Use registerTempTable instead of registerAsTable.", DeprecationWarning)
self.registerTempTable(name)