From 65b814bf50e92e2e9b622d1602f18bacd217181c Mon Sep 17 00:00:00 2001
From: "Joseph K. Bradley" <joseph@databricks.com>
Date: Fri, 9 Sep 2016 05:35:10 -0700
Subject: [PATCH] [SPARK-17456][CORE] Utility for parsing Spark versions
## What changes were proposed in this pull request?
This patch adds methods for extracting major and minor versions as Int types in Scala from a Spark version string.
Motivation: There are many hacks within Spark's codebase to identify and compare Spark versions. We should add a simple utility to standardize these code paths, especially since there have been mistakes made in the past. This will let us add unit tests as well. Currently, I want this functionality to check Spark versions to provide backwards compatibility for ML model persistence.
## How was this patch tested?
Unit tests
Author: Joseph K. Bradley <joseph@databricks.com>
Closes #15017 from jkbradley/version-parsing.
---
.../org/apache/spark/util/VersionUtils.scala | 52 +++++++++++++
.../apache/spark/util/VersionUtilsSuite.scala | 76 +++++++++++++++++++
2 files changed, 128 insertions(+)
create mode 100644 core/src/main/scala/org/apache/spark/util/VersionUtils.scala
create mode 100644 core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
diff --git a/core/src/main/scala/org/apache/spark/util/VersionUtils.scala b/core/src/main/scala/org/apache/spark/util/VersionUtils.scala
new file mode 100644
index 0000000000..828153b868
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/util/VersionUtils.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+/**
+ * Utilities for working with Spark version strings
+ */
+private[spark] object VersionUtils {
+
+ private val majorMinorRegex = """^(\d+)\.(\d+)(\..*)?$""".r
+
+ /**
+ * Given a Spark version string, return the major version number.
+ * E.g., for 2.0.1-SNAPSHOT, return 2.
+ */
+ def majorVersion(sparkVersion: String): Int = majorMinorVersion(sparkVersion)._1
+
+ /**
+ * Given a Spark version string, return the minor version number.
+ * E.g., for 2.0.1-SNAPSHOT, return 0.
+ */
+ def minorVersion(sparkVersion: String): Int = majorMinorVersion(sparkVersion)._2
+
+ /**
+ * Given a Spark version string, return the (major version number, minor version number).
+ * E.g., for 2.0.1-SNAPSHOT, return (2, 0).
+ */
+ def majorMinorVersion(sparkVersion: String): (Int, Int) = {
+ majorMinorRegex.findFirstMatchIn(sparkVersion) match {
+ case Some(m) =>
+ (m.group(1).toInt, m.group(2).toInt)
+ case None =>
+ throw new IllegalArgumentException(s"Spark tried to parse '$sparkVersion' as a Spark" +
+ s" version string, but it could not find the major and minor version numbers.")
+ }
+ }
+}
diff --git a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
new file mode 100644
index 0000000000..aaf79ebd4f
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import org.apache.spark.SparkFunSuite
+
+class VersionUtilsSuite extends SparkFunSuite {
+
+ import org.apache.spark.util.VersionUtils._
+
+ test("Parse Spark major version") {
+ assert(majorVersion("2.0") === 2)
+ assert(majorVersion("12.10.11") === 12)
+ assert(majorVersion("2.0.1-SNAPSHOT") === 2)
+ assert(majorVersion("2.0.x") === 2)
+ withClue("majorVersion parsing should fail for invalid major version number") {
+ intercept[IllegalArgumentException] {
+ majorVersion("2z.0")
+ }
+ }
+ withClue("majorVersion parsing should fail for invalid minor version number") {
+ intercept[IllegalArgumentException] {
+ majorVersion("2.0z")
+ }
+ }
+ }
+
+ test("Parse Spark minor version") {
+ assert(minorVersion("2.0") === 0)
+ assert(minorVersion("12.10.11") === 10)
+ assert(minorVersion("2.0.1-SNAPSHOT") === 0)
+ assert(minorVersion("2.0.x") === 0)
+ withClue("minorVersion parsing should fail for invalid major version number") {
+ intercept[IllegalArgumentException] {
+ minorVersion("2z.0")
+ }
+ }
+ withClue("minorVersion parsing should fail for invalid minor version number") {
+ intercept[IllegalArgumentException] {
+ minorVersion("2.0z")
+ }
+ }
+ }
+
+ test("Parse Spark major and minor versions") {
+ assert(majorMinorVersion("2.0") === (2, 0))
+ assert(majorMinorVersion("12.10.11") === (12, 10))
+ assert(majorMinorVersion("2.0.1-SNAPSHOT") === (2, 0))
+ assert(majorMinorVersion("2.0.x") === (2, 0))
+ withClue("majorMinorVersion parsing should fail for invalid major version number") {
+ intercept[IllegalArgumentException] {
+ majorMinorVersion("2z.0")
+ }
+ }
+ withClue("majorMinorVersion parsing should fail for invalid minor version number") {
+ intercept[IllegalArgumentException] {
+ majorMinorVersion("2.0z")
+ }
+ }
+ }
+}
--
GitLab