Skip to content
Snippets Groups Projects
Commit b47d054c authored by Josh Rosen's avatar Josh Rosen
Browse files

Remove use of abc.ABCMeta due to cloudpickle issue.

cloudpickle runs into issues while pickling subclasses of AccumulatorParam,
which may be related to this Python issue:

    http://bugs.python.org/issue7689

This seems hard to fix and the ABCMeta wasn't necessary, so I removed it.
parent c75ae362
No related branches found
No related tags found
No related merge requests found
...@@ -25,7 +25,8 @@ ...@@ -25,7 +25,8 @@
>>> a.value >>> a.value
13 13
>>> class VectorAccumulatorParam(object): >>> from pyspark.accumulators import AccumulatorParam
>>> class VectorAccumulatorParam(AccumulatorParam):
... def zero(self, value): ... def zero(self, value):
... return [0.0] * len(value) ... return [0.0] * len(value)
... def addInPlace(self, val1, val2): ... def addInPlace(self, val1, val2):
...@@ -61,7 +62,6 @@ Traceback (most recent call last): ...@@ -61,7 +62,6 @@ Traceback (most recent call last):
Exception:... Exception:...
""" """
from abc import ABCMeta, abstractmethod
import struct import struct
import SocketServer import SocketServer
import threading import threading
...@@ -138,23 +138,20 @@ class AccumulatorParam(object): ...@@ -138,23 +138,20 @@ class AccumulatorParam(object):
""" """
Helper object that defines how to accumulate values of a given type. Helper object that defines how to accumulate values of a given type.
""" """
__metaclass__ = ABCMeta
@abstractmethod
def zero(self, value): def zero(self, value):
""" """
Provide a "zero value" for the type, compatible in dimensions with the Provide a "zero value" for the type, compatible in dimensions with the
provided C{value} (e.g., a zero vector) provided C{value} (e.g., a zero vector)
""" """
return raise NotImplementedError
@abstractmethod
def addInPlace(self, value1, value2): def addInPlace(self, value1, value2):
""" """
Add two values of the accumulator's data type, returning a new value; Add two values of the accumulator's data type, returning a new value;
for efficiency, can also update C{value1} in place and return it. for efficiency, can also update C{value1} in place and return it.
""" """
return raise NotImplementedError
class AddingAccumulatorParam(AccumulatorParam): class AddingAccumulatorParam(AccumulatorParam):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment