From 9fd82dbbcb8b10debbe95f1acab53ae8b340f38e Mon Sep 17 00:00:00 2001
From: Davies Liu <davies.liu@gmail.com>
Date: Mon, 4 Aug 2014 15:54:52 -0700
Subject: [PATCH] [SPARK-1687] [PySpark] fix unit tests related to pickable
namedtuple
serializer is imported multiple times during doctests, so it's better to make _hijack_namedtuple() safe to be called multiple times.
Author: Davies Liu <davies.liu@gmail.com>
Closes #1771 from davies/fix and squashes the following commits:
1a9e336 [Davies Liu] fix unit tests
---
python/pyspark/serializers.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py
index 1b52c144df..a10f85b55a 100644
--- a/python/pyspark/serializers.py
+++ b/python/pyspark/serializers.py
@@ -297,8 +297,11 @@ def _hack_namedtuple(cls):
def _hijack_namedtuple():
""" Hack namedtuple() to make it picklable """
- global _old_namedtuple # or it will put in closure
+ # hijack only one time
+ if hasattr(collections.namedtuple, "__hijack"):
+ return
+ global _old_namedtuple # or it will put in closure
def _copy_func(f):
return types.FunctionType(f.func_code, f.func_globals, f.func_name,
f.func_defaults, f.func_closure)
@@ -313,6 +316,7 @@ def _hijack_namedtuple():
collections.namedtuple.func_globals["_old_namedtuple"] = _old_namedtuple
collections.namedtuple.func_globals["_hack_namedtuple"] = _hack_namedtuple
collections.namedtuple.func_code = namedtuple.func_code
+ collections.namedtuple.__hijack = 1
# hack the cls already generated by namedtuple
# those created in other module can be pickled as normal,
--
GitLab