From 6613046c8c2daaf46a8ec13dd0a016aad22af1a4 Mon Sep 17 00:00:00 2001
From: Srinivasa Reddy Vundela <vsr@cloudera.com>
Date: Sun, 30 Apr 2017 21:42:05 -0700
Subject: [PATCH] [MINOR][DOCS][PYTHON] Adding missing boolean type for
replacement value in fillna
## What changes were proposed in this pull request?
Currently pyspark Dataframe.fillna API supports boolean type when we pass dict, but it is missing in documentation.
## How was this patch tested?
>>> spark.createDataFrame([Row(a=True),Row(a=None)]).fillna({"a" : True}).show()
+----+
| a|
+----+
|true|
|true|
+----+
Please review http://spark.apache.org/contributing.html before opening a pull request.
Author: Srinivasa Reddy Vundela <vsr@cloudera.com>
Closes #17688 from vundela/fillna_doc_fix.
---
python/pyspark/sql/dataframe.py | 2 +-
python/pyspark/sql/tests.py | 4 ++++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index ff21bb5d2f..ab6d35bfa7 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -1247,7 +1247,7 @@ class DataFrame(object):
Value to replace null values with.
If the value is a dict, then `subset` is ignored and `value` must be a mapping
from column name (string) to replacement value. The replacement value must be
- an int, long, float, or string.
+ an int, long, float, boolean, or string.
:param subset: optional list of column names to consider.
Columns specified in subset that do not have matching data type are ignored.
For example, if `value` is a string, and subset contains a non-string column,
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 2b2444304e..cd92148dfa 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -1711,6 +1711,10 @@ class SQLTests(ReusedPySparkTestCase):
self.assertEqual(row.age, None)
self.assertEqual(row.height, None)
+ # fillna with dictionary for boolean types
+ row = self.spark.createDataFrame([Row(a=None), Row(a=True)]).fillna({"a": True}).first()
+ self.assertEqual(row.a, True)
+
def test_bitwise_operations(self):
from pyspark.sql import functions
row = Row(a=170, b=75)
--
GitLab