From fdf63f12490c674cc1877ddf7b70343c4fd6f4f1 Mon Sep 17 00:00:00 2001
From: Kevin Conor <kevin@discoverybayconsulting.com>
Date: Fri, 19 Jun 2015 00:12:20 -0700
Subject: [PATCH] [SPARK-8339] [PYSPARK] integer division for python 3
Itertools islice requires an integer for the stop argument. Switching to integer division here prevents a ValueError when vs is evaluated above.
davies
This is my original work, and I license it to the project.
Author: Kevin Conor <kevin@discoverybayconsulting.com>
Closes #6794 from kconor/kconor-patch-1 and squashes the following commits:
da5e700 [Kevin Conor] Integer division for batch size
---
python/pyspark/serializers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py
index d8cdcda3a3..7f9d0a338d 100644
--- a/python/pyspark/serializers.py
+++ b/python/pyspark/serializers.py
@@ -272,7 +272,7 @@ class AutoBatchedSerializer(BatchedSerializer):
if size < best:
batch *= 2
elif size > best * 10 and batch > 1:
- batch /= 2
+ batch //= 2
def __repr__(self):
return "AutoBatchedSerializer(%s)" % self.serializer
--
GitLab