Skip to content
Snippets Groups Projects
Commit f88ac701 authored by Andrew Or's avatar Andrew Or
Browse files

[SPARK-7399] Spark compilation error for scala 2.11

Subsequent fix following #5966. I tried this out locally.

Author: Andrew Or <andrew@databricks.com>

Closes #6129 from andrewor14/211-compilation and squashes the following commits:

713868f [Andrew Or] Fix compilation issue for scala 2.11
parent f6e18388
No related branches found
No related tags found
No related merge requests found
...@@ -1524,7 +1524,7 @@ abstract class RDD[T: ClassTag]( ...@@ -1524,7 +1524,7 @@ abstract class RDD[T: ClassTag](
* doCheckpoint() is called recursively on the parent RDDs. * doCheckpoint() is called recursively on the parent RDDs.
*/ */
private[spark] def doCheckpoint(): Unit = { private[spark] def doCheckpoint(): Unit = {
RDDOperationScope.withScope(sc, "checkpoint", false, true) { RDDOperationScope.withScope(sc, "checkpoint", allowNesting = false, ignoreParent = true) {
if (!doCheckpointCalled) { if (!doCheckpointCalled) {
doCheckpointCalled = true doCheckpointCalled = true
if (checkpointData.isDefined) { if (checkpointData.isDefined) {
......
...@@ -96,7 +96,7 @@ private[spark] object RDDOperationScope { ...@@ -96,7 +96,7 @@ private[spark] object RDDOperationScope {
sc: SparkContext, sc: SparkContext,
allowNesting: Boolean = false)(body: => T): T = { allowNesting: Boolean = false)(body: => T): T = {
val callerMethodName = Thread.currentThread.getStackTrace()(3).getMethodName val callerMethodName = Thread.currentThread.getStackTrace()(3).getMethodName
withScope[T](sc, callerMethodName, allowNesting)(body) withScope[T](sc, callerMethodName, allowNesting, ignoreParent = false)(body)
} }
/** /**
...@@ -116,7 +116,7 @@ private[spark] object RDDOperationScope { ...@@ -116,7 +116,7 @@ private[spark] object RDDOperationScope {
sc: SparkContext, sc: SparkContext,
name: String, name: String,
allowNesting: Boolean, allowNesting: Boolean,
ignoreParent: Boolean = false)(body: => T): T = { ignoreParent: Boolean)(body: => T): T = {
// Save the old scope to restore it later // Save the old scope to restore it later
val scopeKey = SparkContext.RDD_SCOPE_KEY val scopeKey = SparkContext.RDD_SCOPE_KEY
val noOverrideKey = SparkContext.RDD_SCOPE_NO_OVERRIDE_KEY val noOverrideKey = SparkContext.RDD_SCOPE_NO_OVERRIDE_KEY
......
...@@ -61,11 +61,11 @@ class RDDOperationScopeSuite extends FunSuite with BeforeAndAfter { ...@@ -61,11 +61,11 @@ class RDDOperationScopeSuite extends FunSuite with BeforeAndAfter {
var rdd1: MyCoolRDD = null var rdd1: MyCoolRDD = null
var rdd2: MyCoolRDD = null var rdd2: MyCoolRDD = null
var rdd3: MyCoolRDD = null var rdd3: MyCoolRDD = null
RDDOperationScope.withScope(sc, "scope1", allowNesting = false) { RDDOperationScope.withScope(sc, "scope1", allowNesting = false, ignoreParent = false) {
rdd1 = new MyCoolRDD(sc) rdd1 = new MyCoolRDD(sc)
RDDOperationScope.withScope(sc, "scope2", allowNesting = false) { RDDOperationScope.withScope(sc, "scope2", allowNesting = false, ignoreParent = false) {
rdd2 = new MyCoolRDD(sc) rdd2 = new MyCoolRDD(sc)
RDDOperationScope.withScope(sc, "scope3", allowNesting = false) { RDDOperationScope.withScope(sc, "scope3", allowNesting = false, ignoreParent = false) {
rdd3 = new MyCoolRDD(sc) rdd3 = new MyCoolRDD(sc)
} }
} }
...@@ -84,11 +84,13 @@ class RDDOperationScopeSuite extends FunSuite with BeforeAndAfter { ...@@ -84,11 +84,13 @@ class RDDOperationScopeSuite extends FunSuite with BeforeAndAfter {
var rdd1: MyCoolRDD = null var rdd1: MyCoolRDD = null
var rdd2: MyCoolRDD = null var rdd2: MyCoolRDD = null
var rdd3: MyCoolRDD = null var rdd3: MyCoolRDD = null
RDDOperationScope.withScope(sc, "scope1", allowNesting = true) { // allow nesting here // allow nesting here
RDDOperationScope.withScope(sc, "scope1", allowNesting = true, ignoreParent = false) {
rdd1 = new MyCoolRDD(sc) rdd1 = new MyCoolRDD(sc)
RDDOperationScope.withScope(sc, "scope2", allowNesting = false) { // stop nesting here // stop nesting here
RDDOperationScope.withScope(sc, "scope2", allowNesting = false, ignoreParent = false) {
rdd2 = new MyCoolRDD(sc) rdd2 = new MyCoolRDD(sc)
RDDOperationScope.withScope(sc, "scope3", allowNesting = false) { RDDOperationScope.withScope(sc, "scope3", allowNesting = false, ignoreParent = false) {
rdd3 = new MyCoolRDD(sc) rdd3 = new MyCoolRDD(sc)
} }
} }
...@@ -107,11 +109,11 @@ class RDDOperationScopeSuite extends FunSuite with BeforeAndAfter { ...@@ -107,11 +109,11 @@ class RDDOperationScopeSuite extends FunSuite with BeforeAndAfter {
var rdd1: MyCoolRDD = null var rdd1: MyCoolRDD = null
var rdd2: MyCoolRDD = null var rdd2: MyCoolRDD = null
var rdd3: MyCoolRDD = null var rdd3: MyCoolRDD = null
RDDOperationScope.withScope(sc, "scope1", allowNesting = true) { RDDOperationScope.withScope(sc, "scope1", allowNesting = true, ignoreParent = false) {
rdd1 = new MyCoolRDD(sc) rdd1 = new MyCoolRDD(sc)
RDDOperationScope.withScope(sc, "scope2", allowNesting = true) { RDDOperationScope.withScope(sc, "scope2", allowNesting = true, ignoreParent = false) {
rdd2 = new MyCoolRDD(sc) rdd2 = new MyCoolRDD(sc)
RDDOperationScope.withScope(sc, "scope3", allowNesting = true) { RDDOperationScope.withScope(sc, "scope3", allowNesting = true, ignoreParent = false) {
rdd3 = new MyCoolRDD(sc) rdd3 = new MyCoolRDD(sc)
} }
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment