Skip to content
Snippets Groups Projects
Commit 087e31a7 authored by andrewor14's avatar andrewor14 Committed by Andrew Or
Browse files

[HOT FIX] Yarn stable tests don't compile

This is caused by this commit: acd4ac7c

Author: andrewor14 <andrew@databricks.com>
Author: Andrew Or <andrew@databricks.com>

Closes #3041 from andrewor14/yarn-hot-fix and squashes the following commits:

e5deba1 [andrewor14] Add new line at the end (minor)
aa998e8 [Andrew Or] Compilation hot fix
parent 55ab7770
No related branches found
No related tags found
No related merge requests found
...@@ -64,6 +64,8 @@ private[yarn] abstract class YarnAllocator( ...@@ -64,6 +64,8 @@ private[yarn] abstract class YarnAllocator(
securityMgr: SecurityManager) securityMgr: SecurityManager)
extends Logging { extends Logging {
import YarnAllocator._
// These three are locked on allocatedHostToContainersMap. Complementary data structures // These three are locked on allocatedHostToContainersMap. Complementary data structures
// allocatedHostToContainersMap : containers which are running : host, Set<containerid> // allocatedHostToContainersMap : containers which are running : host, Set<containerid>
// allocatedContainerToHostMap: container to host mapping. // allocatedContainerToHostMap: container to host mapping.
...@@ -439,19 +441,6 @@ private[yarn] abstract class YarnAllocator( ...@@ -439,19 +441,6 @@ private[yarn] abstract class YarnAllocator(
} }
} }
private val MEM_REGEX = "[0-9.]+ [KMG]B"
private val PMEM_EXCEEDED_PATTERN =
Pattern.compile(s"$MEM_REGEX of $MEM_REGEX physical memory used")
private val VMEM_EXCEEDED_PATTERN =
Pattern.compile(s"$MEM_REGEX of $MEM_REGEX virtual memory used")
def memLimitExceededLogMessage(diagnostics: String, pattern: Pattern): String = {
val matcher = pattern.matcher(diagnostics)
val diag = if (matcher.find()) " " + matcher.group() + "." else ""
("Container killed by YARN for exceeding memory limits." + diag
+ " Consider boosting spark.yarn.executor.memoryOverhead.")
}
protected def allocatedContainersOnHost(host: String): Int = { protected def allocatedContainersOnHost(host: String): Int = {
var retval = 0 var retval = 0
allocatedHostToContainersMap.synchronized { allocatedHostToContainersMap.synchronized {
...@@ -532,3 +521,18 @@ private[yarn] abstract class YarnAllocator( ...@@ -532,3 +521,18 @@ private[yarn] abstract class YarnAllocator(
} }
} }
private object YarnAllocator {
val MEM_REGEX = "[0-9.]+ [KMG]B"
val PMEM_EXCEEDED_PATTERN =
Pattern.compile(s"$MEM_REGEX of $MEM_REGEX physical memory used")
val VMEM_EXCEEDED_PATTERN =
Pattern.compile(s"$MEM_REGEX of $MEM_REGEX virtual memory used")
def memLimitExceededLogMessage(diagnostics: String, pattern: Pattern): String = {
val matcher = pattern.matcher(diagnostics)
val diag = if (matcher.find()) " " + matcher.group() + "." else ""
("Container killed by YARN for exceeding memory limits." + diag
+ " Consider boosting spark.yarn.executor.memoryOverhead.")
}
}
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
package org.apache.spark.deploy.yarn package org.apache.spark.deploy.yarn
import org.apache.spark.deploy.yarn.MemLimitLogger._ import org.apache.spark.deploy.yarn.YarnAllocator._
import org.scalatest.FunSuite import org.scalatest.FunSuite
class YarnAllocatorSuite extends FunSuite { class YarnAllocatorSuite extends FunSuite {
...@@ -31,4 +31,4 @@ class YarnAllocatorSuite extends FunSuite { ...@@ -31,4 +31,4 @@ class YarnAllocatorSuite extends FunSuite {
assert(vmemMsg.contains("5.8 GB of 4.2 GB virtual memory used.")) assert(vmemMsg.contains("5.8 GB of 4.2 GB virtual memory used."))
assert(pmemMsg.contains("2.1 MB of 2 GB physical memory used.")) assert(pmemMsg.contains("2.1 MB of 2 GB physical memory used."))
} }
} }
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment