Skip to content
Snippets Groups Projects
Commit 9a54de16 authored by Erik Erlandson's avatar Erik Erlandson Committed by Reynold Xin
Browse files

[SPARK-2911]: provide rdd.parent[T](j) to obtain jth parent RDD

Author: Erik Erlandson <eerlands@redhat.com>

Closes #1841 from erikerlandson/spark-2911-pr and squashes the following commits:

4699e2f [Erik Erlandson] [SPARK-2911]: provide rdd.parent[T](j) to obtain jth parent RDD
parent 9de6a42b
No related branches found
No related tags found
No related merge requests found
...@@ -1233,6 +1233,11 @@ abstract class RDD[T: ClassTag]( ...@@ -1233,6 +1233,11 @@ abstract class RDD[T: ClassTag](
dependencies.head.rdd.asInstanceOf[RDD[U]] dependencies.head.rdd.asInstanceOf[RDD[U]]
} }
/** Returns the jth parent RDD: e.g. rdd.parent[T](0) is equivalent to rdd.firstParent[T] */
protected[spark] def parent[U: ClassTag](j: Int) = {
dependencies(j).rdd.asInstanceOf[RDD[U]]
}
/** The [[org.apache.spark.SparkContext]] that this RDD was created on. */ /** The [[org.apache.spark.SparkContext]] that this RDD was created on. */
def context = sc def context = sc
......
...@@ -726,6 +726,16 @@ class RDDSuite extends FunSuite with SharedSparkContext { ...@@ -726,6 +726,16 @@ class RDDSuite extends FunSuite with SharedSparkContext {
jrdd.rdd.retag.collect() jrdd.rdd.retag.collect()
} }
test("parent method") {
val rdd1 = sc.parallelize(1 to 10, 2)
val rdd2 = rdd1.filter(_ % 2 == 0)
val rdd3 = rdd2.map(_ + 1)
val rdd4 = new UnionRDD(sc, List(rdd1, rdd2, rdd3))
assert(rdd4.parent(0).isInstanceOf[ParallelCollectionRDD[_]])
assert(rdd4.parent(1).isInstanceOf[FilteredRDD[_]])
assert(rdd4.parent(2).isInstanceOf[MappedRDD[_, _]])
}
test("getNarrowAncestors") { test("getNarrowAncestors") {
val rdd1 = sc.parallelize(1 to 100, 4) val rdd1 = sc.parallelize(1 to 100, 4)
val rdd2 = rdd1.filter(_ % 2 == 0).map(_ + 1) val rdd2 = rdd1.filter(_ % 2 == 0).map(_ + 1)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment