From d874f8b546d8fae95bc92d8461b8189e51cb731b Mon Sep 17 00:00:00 2001
From: linweizhong <linweizhong@huawei.com>
Date: Fri, 24 Apr 2015 20:23:19 -0700
Subject: [PATCH] [PySpark][Minor] Update sql example, so that can read file
correctly
To run Spark, default will read file from HDFS if we don't set the schema.
Author: linweizhong <linweizhong@huawei.com>
Closes #5684 from Sephiroth-Lin/pyspark_example_minor and squashes the following commits:
19fe145 [linweizhong] Update example sql.py, so that can read file correctly
---
examples/src/main/python/sql.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/examples/src/main/python/sql.py b/examples/src/main/python/sql.py
index 87d7b088f0..2c18875932 100644
--- a/examples/src/main/python/sql.py
+++ b/examples/src/main/python/sql.py
@@ -18,6 +18,7 @@
from __future__ import print_function
import os
+import sys
from pyspark import SparkContext
from pyspark.sql import SQLContext
@@ -50,7 +51,11 @@ if __name__ == "__main__":
# A JSON dataset is pointed to by path.
# The path can be either a single text file or a directory storing text files.
- path = os.path.join(os.environ['SPARK_HOME'], "examples/src/main/resources/people.json")
+ if len(sys.argv) < 2:
+ path = "file://" + \
+ os.path.join(os.environ['SPARK_HOME'], "examples/src/main/resources/people.json")
+ else:
+ path = sys.argv[1]
# Create a DataFrame from the file(s) pointed to by path
people = sqlContext.jsonFile(path)
# root
--
GitLab