From 747c2ba8006d5b86f3be8dfa9ace639042a35628 Mon Sep 17 00:00:00 2001
From: Yanbo Liang <ybliang8@gmail.com>
Date: Tue, 18 Aug 2015 12:56:36 -0700
Subject: [PATCH] [SPARK-10032] [PYSPARK] [DOC] Add Python example for mllib
LDAModel user guide
Add Python example for mllib LDAModel user guide
Author: Yanbo Liang <ybliang8@gmail.com>
Closes #8227 from yanboliang/spark-10032.
---
docs/mllib-clustering.md | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/docs/mllib-clustering.md b/docs/mllib-clustering.md
index bb875ae2ae..fd9ab258e1 100644
--- a/docs/mllib-clustering.md
+++ b/docs/mllib-clustering.md
@@ -564,6 +564,34 @@ public class JavaLDAExample {
{% endhighlight %}
</div>
+<div data-lang="python" markdown="1">
+{% highlight python %}
+from pyspark.mllib.clustering import LDA, LDAModel
+from pyspark.mllib.linalg import Vectors
+
+# Load and parse the data
+data = sc.textFile("data/mllib/sample_lda_data.txt")
+parsedData = data.map(lambda line: Vectors.dense([float(x) for x in line.strip().split(' ')]))
+# Index documents with unique IDs
+corpus = parsedData.zipWithIndex().map(lambda x: [x[1], x[0]]).cache()
+
+# Cluster the documents into three topics using LDA
+ldaModel = LDA.train(corpus, k=3)
+
+# Output topics. Each is a distribution over words (matching word count vectors)
+print("Learned topics (as distributions over vocab of " + str(ldaModel.vocabSize()) + " words):")
+topics = ldaModel.topicsMatrix()
+for topic in range(3):
+ print("Topic " + str(topic) + ":")
+ for word in range(0, ldaModel.vocabSize()):
+ print(" " + str(topics[word][topic]))
+
+# Save and load model
+model.save(sc, "myModelPath")
+sameModel = LDAModel.load(sc, "myModelPath")
+{% endhighlight %}
+</div>
+
</div>
## Streaming k-means
--
GitLab