Skip to content
Snippets Groups Projects
Commit 948c2390 authored by Sean Owen's avatar Sean Owen
Browse files

SPARK-3357 [CORE] Internal log messages should be set at DEBUG level instead of INFO

Demote some 'noisy' log messages to debug level. I added a few more, to include everything that gets logged in stanzas like this:

```
15/03/01 00:03:54 INFO BlockManager: Removing broadcast 0
15/03/01 00:03:54 INFO BlockManager: Removing block broadcast_0_piece0
15/03/01 00:03:54 INFO MemoryStore: Block broadcast_0_piece0 of size 839 dropped from memory (free 277976091)
15/03/01 00:03:54 INFO BlockManagerInfo: Removed broadcast_0_piece0 on localhost:49524 in memory (size: 839.0 B, free: 265.1 MB)
15/03/01 00:03:54 INFO BlockManagerMaster: Updated info of block broadcast_0_piece0
15/03/01 00:03:54 INFO BlockManager: Removing block broadcast_0
15/03/01 00:03:54 INFO MemoryStore: Block broadcast_0 of size 1088 dropped from memory (free 277977179)
15/03/01 00:03:54 INFO ContextCleaner: Cleaned broadcast 0
```

as well as regular messages like

```
15/03/01 00:02:33 INFO MemoryStore: ensureFreeSpace(2640) called with curMem=47322, maxMem=278019440
```

WDYT? good or should some be left alone?

CC mengxr who suggested some of this.

Author: Sean Owen <sowen@cloudera.com>

Closes #4838 from srowen/SPARK-3357 and squashes the following commits:

dce75c1 [Sean Owen] Back out some debug level changes
d9b784d [Sean Owen] Demote some 'noisy' log messages to debug level
parent d8fb40ed
No related branches found
No related tags found
No related merge requests found
...@@ -188,10 +188,10 @@ private[spark] class ContextCleaner(sc: SparkContext) extends Logging { ...@@ -188,10 +188,10 @@ private[spark] class ContextCleaner(sc: SparkContext) extends Logging {
/** Perform broadcast cleanup. */ /** Perform broadcast cleanup. */
def doCleanupBroadcast(broadcastId: Long, blocking: Boolean) { def doCleanupBroadcast(broadcastId: Long, blocking: Boolean) {
try { try {
logDebug("Cleaning broadcast " + broadcastId) logDebug(s"Cleaning broadcast $broadcastId")
broadcastManager.unbroadcast(broadcastId, true, blocking) broadcastManager.unbroadcast(broadcastId, true, blocking)
listeners.foreach(_.broadcastCleaned(broadcastId)) listeners.foreach(_.broadcastCleaned(broadcastId))
logInfo("Cleaned broadcast " + broadcastId) logDebug(s"Cleaned broadcast $broadcastId")
} catch { } catch {
case e: Exception => logError("Error cleaning broadcast " + broadcastId, e) case e: Exception => logError("Error cleaning broadcast " + broadcastId, e)
} }
......
...@@ -1074,7 +1074,7 @@ private[spark] class BlockManager( ...@@ -1074,7 +1074,7 @@ private[spark] class BlockManager(
* Remove all blocks belonging to the given broadcast. * Remove all blocks belonging to the given broadcast.
*/ */
def removeBroadcast(broadcastId: Long, tellMaster: Boolean): Int = { def removeBroadcast(broadcastId: Long, tellMaster: Boolean): Int = {
logInfo(s"Removing broadcast $broadcastId") logDebug(s"Removing broadcast $broadcastId")
val blocksToRemove = blockInfo.keys.collect { val blocksToRemove = blockInfo.keys.collect {
case bid @ BroadcastBlockId(`broadcastId`, _) => bid case bid @ BroadcastBlockId(`broadcastId`, _) => bid
} }
...@@ -1086,7 +1086,7 @@ private[spark] class BlockManager( ...@@ -1086,7 +1086,7 @@ private[spark] class BlockManager(
* Remove a block from both memory and disk. * Remove a block from both memory and disk.
*/ */
def removeBlock(blockId: BlockId, tellMaster: Boolean = true): Unit = { def removeBlock(blockId: BlockId, tellMaster: Boolean = true): Unit = {
logInfo(s"Removing block $blockId") logDebug(s"Removing block $blockId")
val info = blockInfo.get(blockId).orNull val info = blockInfo.get(blockId).orNull
if (info != null) { if (info != null) {
info.synchronized { info.synchronized {
......
...@@ -61,7 +61,7 @@ class BlockManagerMaster( ...@@ -61,7 +61,7 @@ class BlockManagerMaster(
tachyonSize: Long): Boolean = { tachyonSize: Long): Boolean = {
val res = askDriverWithReply[Boolean]( val res = askDriverWithReply[Boolean](
UpdateBlockInfo(blockManagerId, blockId, storageLevel, memSize, diskSize, tachyonSize)) UpdateBlockInfo(blockManagerId, blockId, storageLevel, memSize, diskSize, tachyonSize))
logInfo("Updated info of block " + blockId) logDebug(s"Updated info of block $blockId")
res res
} }
......
...@@ -184,7 +184,7 @@ private[spark] class MemoryStore(blockManager: BlockManager, maxMemory: Long) ...@@ -184,7 +184,7 @@ private[spark] class MemoryStore(blockManager: BlockManager, maxMemory: Long)
val entry = entries.remove(blockId) val entry = entries.remove(blockId)
if (entry != null) { if (entry != null) {
currentMemory -= entry.size currentMemory -= entry.size
logInfo(s"Block $blockId of size ${entry.size} dropped from memory (free $freeMemory)") logDebug(s"Block $blockId of size ${entry.size} dropped from memory (free $freeMemory)")
true true
} else { } else {
false false
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment