From b47d054cfc5ef45b92a1c970388722ffa0283e66 Mon Sep 17 00:00:00 2001
From: Josh Rosen <joshrosen@eecs.berkeley.edu>
Date: Wed, 23 Jan 2013 11:18:25 -0800
Subject: [PATCH] Remove use of abc.ABCMeta due to cloudpickle issue.
cloudpickle runs into issues while pickling subclasses of AccumulatorParam,
which may be related to this Python issue:
http://bugs.python.org/issue7689
This seems hard to fix and the ABCMeta wasn't necessary, so I removed it.
---
python/pyspark/accumulators.py | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/python/pyspark/accumulators.py b/python/pyspark/accumulators.py
index 5a9269f9bb..61fcbbd376 100644
--- a/python/pyspark/accumulators.py
+++ b/python/pyspark/accumulators.py
@@ -25,7 +25,8 @@
>>> a.value
13
->>> class VectorAccumulatorParam(object):
+>>> from pyspark.accumulators import AccumulatorParam
+>>> class VectorAccumulatorParam(AccumulatorParam):
... def zero(self, value):
... return [0.0] * len(value)
... def addInPlace(self, val1, val2):
@@ -61,7 +62,6 @@ Traceback (most recent call last):
Exception:...
"""
-from abc import ABCMeta, abstractmethod
import struct
import SocketServer
import threading
@@ -138,23 +138,20 @@ class AccumulatorParam(object):
"""
Helper object that defines how to accumulate values of a given type.
"""
- __metaclass__ = ABCMeta
- @abstractmethod
def zero(self, value):
"""
Provide a "zero value" for the type, compatible in dimensions with the
provided C{value} (e.g., a zero vector)
"""
- return
+ raise NotImplementedError
- @abstractmethod
def addInPlace(self, value1, value2):
"""
Add two values of the accumulator's data type, returning a new value;
for efficiency, can also update C{value1} in place and return it.
"""
- return
+ raise NotImplementedError
class AddingAccumulatorParam(AccumulatorParam):
--
GitLab