From 0fd84b05dc9ac3de240791e2d4200d8bdffbb01a Mon Sep 17 00:00:00 2001
From: 10129659 <chen.yanshan@zte.com.cn>
Date: Fri, 26 May 2017 18:03:23 +0100
Subject: [PATCH] [SPARK-20835][CORE] It should exit directly when the
--total-executor-cores parameter is setted less than 0 when submit a
application
## What changes were proposed in this pull request?
In my test, the submitted app running with out an error when the --total-executor-cores less than 0
and given the warnings:
"2017-05-22 17:19:36,319 WARN org.apache.spark.scheduler.TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources";
It should exit directly when the --total-executor-cores parameter is setted less than 0 when submit a application
(Please fill in changes proposed in this fix)
## How was this patch tested?
Run the ut tests
(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
(If this patch involves UI changes, please attach a screenshot; otherwise, remove this)
Please review http://spark.apache.org/contributing.html before opening a pull request.
Author: 10129659 <chen.yanshan@zte.com.cn>
Closes #18060 from eatoncys/totalcores.
---
.../spark/deploy/SparkSubmitArguments.scala | 20 +++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 0144fd1056..5100a17006 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -27,11 +27,14 @@ import java.util.jar.JarFile
import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, HashMap}
import scala.io.Source
+import scala.util.Try
import org.apache.spark.deploy.SparkSubmitAction._
import org.apache.spark.launcher.SparkSubmitArgumentsParser
+import org.apache.spark.network.util.JavaUtils
import org.apache.spark.util.Utils
+
/**
* Parses and encapsulates arguments from the spark-submit script.
* The env argument is used for testing.
@@ -253,6 +256,23 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
if (mainClass == null && SparkSubmit.isUserJar(primaryResource)) {
SparkSubmit.printErrorAndExit("No main class set in JAR; please specify one with --class")
}
+ if (driverMemory != null
+ && Try(JavaUtils.byteStringAsBytes(driverMemory)).getOrElse(-1L) <= 0) {
+ SparkSubmit.printErrorAndExit("Driver Memory must be a positive number")
+ }
+ if (executorMemory != null
+ && Try(JavaUtils.byteStringAsBytes(executorMemory)).getOrElse(-1L) <= 0) {
+ SparkSubmit.printErrorAndExit("Executor Memory cores must be a positive number")
+ }
+ if (executorCores != null && Try(executorCores.toInt).getOrElse(-1) <= 0) {
+ SparkSubmit.printErrorAndExit("Executor cores must be a positive number")
+ }
+ if (totalExecutorCores != null && Try(totalExecutorCores.toInt).getOrElse(-1) <= 0) {
+ SparkSubmit.printErrorAndExit("Total executor cores must be a positive number")
+ }
+ if (numExecutors != null && Try(numExecutors.toInt).getOrElse(-1) <= 0) {
+ SparkSubmit.printErrorAndExit("Number of executors must be a positive number")
+ }
if (pyFiles != null && !isPython) {
SparkSubmit.printErrorAndExit("--py-files given but primary resource is not a Python script")
}
--
GitLab