diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala index 64f2f00484f479fc3e2a13388ddeb3f6f3d871b2..f9e24167a17ec88e7639ee373765a0f20c4bfcba 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSinkLog.scala @@ -84,14 +84,11 @@ class FileStreamSinkLog( private implicit val formats = Serialization.formats(NoTypeHints) - protected override val fileCleanupDelayMs = - sparkSession.conf.get(SQLConf.FILE_SINK_LOG_CLEANUP_DELAY) + protected override val fileCleanupDelayMs = sparkSession.sessionState.conf.fileSinkLogCleanupDelay - protected override val isDeletingExpiredLog = - sparkSession.conf.get(SQLConf.FILE_SINK_LOG_DELETION) + protected override val isDeletingExpiredLog = sparkSession.sessionState.conf.fileSinkLogDeletion - protected override val compactInterval = - sparkSession.conf.get(SQLConf.FILE_SINK_LOG_COMPACT_INTERVAL) + protected override val compactInterval = sparkSession.sessionState.conf.fileSinkLogCompactInterval require(compactInterval > 0, s"Please set ${SQLConf.FILE_SINK_LOG_COMPACT_INTERVAL.key} (was $compactInterval) " + "to a positive value.") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala index 8103309aff2ae8c6d04069d597fac451f6ce5fab..4681f2ba08c84f90626f2e21b81e89be1d652c00 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSourceLog.scala @@ -39,16 +39,15 @@ class FileStreamSourceLog( // Configurations about metadata compaction protected override val compactInterval = - sparkSession.conf.get(SQLConf.FILE_SOURCE_LOG_COMPACT_INTERVAL) + sparkSession.sessionState.conf.fileSourceLogCompactInterval require(compactInterval > 0, s"Please set ${SQLConf.FILE_SOURCE_LOG_COMPACT_INTERVAL.key} (was $compactInterval) to a " + s"positive value.") protected override val fileCleanupDelayMs = - sparkSession.conf.get(SQLConf.FILE_SOURCE_LOG_CLEANUP_DELAY) + sparkSession.sessionState.conf.fileSourceLogCleanupDelay - protected override val isDeletingExpiredLog = - sparkSession.conf.get(SQLConf.FILE_SOURCE_LOG_DELETION) + protected override val isDeletingExpiredLog = sparkSession.sessionState.conf.fileSourceLogDeletion private implicit val formats = Serialization.formats(NoTypeHints) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index f8b7a7f8ef77f503bbe0a844f96ee002e63ae469..e67140fefef9ad8b28075b3b675b364d62a82fbf 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -620,10 +620,16 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging { def fileSinkLogDeletion: Boolean = getConf(FILE_SINK_LOG_DELETION) - def fileSinkLogCompatInterval: Int = getConf(FILE_SINK_LOG_COMPACT_INTERVAL) + def fileSinkLogCompactInterval: Int = getConf(FILE_SINK_LOG_COMPACT_INTERVAL) def fileSinkLogCleanupDelay: Long = getConf(FILE_SINK_LOG_CLEANUP_DELAY) + def fileSourceLogDeletion: Boolean = getConf(FILE_SOURCE_LOG_DELETION) + + def fileSourceLogCompactInterval: Int = getConf(FILE_SOURCE_LOG_COMPACT_INTERVAL) + + def fileSourceLogCleanupDelay: Long = getConf(FILE_SOURCE_LOG_CLEANUP_DELAY) + def streamingSchemaInference: Boolean = getConf(STREAMING_SCHEMA_INFERENCE) def streamingPollingDelay: Long = getConf(STREAMING_POLLING_DELAY)