diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala
new file mode 100644
index 0000000000000000000000000000000000000000..04467342e6ab529d2f24a5a2b1de8a5f2e28812b
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import scala.language.implicitConversions
+import scala.util.parsing.combinator.lexical.StdLexical
+import scala.util.parsing.combinator.syntactical.StandardTokenParsers
+import scala.util.parsing.combinator.{PackratParsers, RegexParsers}
+import scala.util.parsing.input.CharArrayReader.EofCh
+
+import org.apache.spark.sql.catalyst.plans.logical._
+
+private[sql] abstract class AbstractSparkSQLParser
+ extends StandardTokenParsers with PackratParsers {
+
+ def apply(input: String): LogicalPlan = phrase(start)(new lexical.Scanner(input)) match {
+ case Success(plan, _) => plan
+ case failureOrError => sys.error(failureOrError.toString)
+ }
+
+ protected case class Keyword(str: String)
+
+ protected def start: Parser[LogicalPlan]
+
+ // Returns the whole input string
+ protected lazy val wholeInput: Parser[String] = new Parser[String] {
+ def apply(in: Input): ParseResult[String] =
+ Success(in.source.toString, in.drop(in.source.length()))
+ }
+
+ // Returns the rest of the input string that are not parsed yet
+ protected lazy val restInput: Parser[String] = new Parser[String] {
+ def apply(in: Input): ParseResult[String] =
+ Success(
+ in.source.subSequence(in.offset, in.source.length()).toString,
+ in.drop(in.source.length()))
+ }
+}
+
+class SqlLexical(val keywords: Seq[String]) extends StdLexical {
+ case class FloatLit(chars: String) extends Token {
+ override def toString = chars
+ }
+
+ reserved ++= keywords.flatMap(w => allCaseVersions(w))
+
+ delimiters += (
+ "@", "*", "+", "-", "<", "=", "<>", "!=", "<=", ">=", ">", "/", "(", ")",
+ ",", ";", "%", "{", "}", ":", "[", "]", "."
+ )
+
+ override lazy val token: Parser[Token] =
+ ( identChar ~ (identChar | digit).* ^^
+ { case first ~ rest => processIdent((first :: rest).mkString) }
+ | rep1(digit) ~ ('.' ~> digit.*).? ^^ {
+ case i ~ None => NumericLit(i.mkString)
+ case i ~ Some(d) => FloatLit(i.mkString + "." + d.mkString)
+ }
+ | '\'' ~> chrExcept('\'', '\n', EofCh).* <~ '\'' ^^
+ { case chars => StringLit(chars mkString "") }
+ | '"' ~> chrExcept('"', '\n', EofCh).* <~ '"' ^^
+ { case chars => StringLit(chars mkString "") }
+ | EofCh ^^^ EOF
+ | '\'' ~> failure("unclosed string literal")
+ | '"' ~> failure("unclosed string literal")
+ | delim
+ | failure("illegal character")
+ )
+
+ override def identChar = letter | elem('_')
+
+ override def whitespace: Parser[Any] =
+ ( whitespaceChar
+ | '/' ~ '*' ~ comment
+ | '/' ~ '/' ~ chrExcept(EofCh, '\n').*
+ | '#' ~ chrExcept(EofCh, '\n').*
+ | '-' ~ '-' ~ chrExcept(EofCh, '\n').*
+ | '/' ~ '*' ~ failure("unclosed comment")
+ ).*
+
+ /** Generate all variations of upper and lower case of a given string */
+ def allCaseVersions(s: String, prefix: String = ""): Stream[String] = {
+ if (s == "") {
+ Stream(prefix)
+ } else {
+ allCaseVersions(s.tail, prefix + s.head.toLower) ++
+ allCaseVersions(s.tail, prefix + s.head.toUpper)
+ }
+ }
+}
+
+/**
+ * The top level Spark SQL parser. This parser recognizes syntaxes that are available for all SQL
+ * dialects supported by Spark SQL, and delegates all the other syntaxes to the `fallback` parser.
+ *
+ * @param fallback A function that parses an input string to a logical plan
+ */
+private[sql] class SparkSQLParser(fallback: String => LogicalPlan) extends AbstractSparkSQLParser {
+
+ // A parser for the key-value part of the "SET [key = [value ]]" syntax
+ private object SetCommandParser extends RegexParsers {
+ private val key: Parser[String] = "(?m)[^=]+".r
+
+ private val value: Parser[String] = "(?m).*$".r
+
+ private val pair: Parser[LogicalPlan] =
+ (key ~ ("=".r ~> value).?).? ^^ {
+ case None => SetCommand(None)
+ case Some(k ~ v) => SetCommand(Some(k.trim -> v.map(_.trim)))
+ }
+
+ def apply(input: String): LogicalPlan = parseAll(pair, input) match {
+ case Success(plan, _) => plan
+ case x => sys.error(x.toString)
+ }
+ }
+
+ protected val AS = Keyword("AS")
+ protected val CACHE = Keyword("CACHE")
+ protected val LAZY = Keyword("LAZY")
+ protected val SET = Keyword("SET")
+ protected val TABLE = Keyword("TABLE")
+ protected val SOURCE = Keyword("SOURCE")
+ protected val UNCACHE = Keyword("UNCACHE")
+
+ protected implicit def asParser(k: Keyword): Parser[String] =
+ lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
+
+ private val reservedWords: Seq[String] =
+ this
+ .getClass
+ .getMethods
+ .filter(_.getReturnType == classOf[Keyword])
+ .map(_.invoke(this).asInstanceOf[Keyword].str)
+
+ override val lexical = new SqlLexical(reservedWords)
+
+ override protected lazy val start: Parser[LogicalPlan] =
+ cache | uncache | set | shell | source | others
+
+ private lazy val cache: Parser[LogicalPlan] =
+ CACHE ~> LAZY.? ~ (TABLE ~> ident) ~ (AS ~> restInput).? ^^ {
+ case isLazy ~ tableName ~ plan =>
+ CacheTableCommand(tableName, plan.map(fallback), isLazy.isDefined)
+ }
+
+ private lazy val uncache: Parser[LogicalPlan] =
+ UNCACHE ~ TABLE ~> ident ^^ {
+ case tableName => UncacheTableCommand(tableName)
+ }
+
+ private lazy val set: Parser[LogicalPlan] =
+ SET ~> restInput ^^ {
+ case input => SetCommandParser(input)
+ }
+
+ private lazy val shell: Parser[LogicalPlan] =
+ "!" ~> restInput ^^ {
+ case input => ShellCommand(input.trim)
+ }
+
+ private lazy val source: Parser[LogicalPlan] =
+ SOURCE ~> restInput ^^ {
+ case input => SourceCommand(input.trim)
+ }
+
+ private lazy val others: Parser[LogicalPlan] =
+ wholeInput ^^ {
+ case input => fallback(input)
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
index 4662f585cfe151e8522973e9bdab4ae55bd28bd0..b4d606d37e732f9875759f7bd0c33af7a7e86b64 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
@@ -18,10 +18,6 @@
package org.apache.spark.sql.catalyst
import scala.language.implicitConversions
-import scala.util.parsing.combinator.lexical.StdLexical
-import scala.util.parsing.combinator.syntactical.StandardTokenParsers
-import scala.util.parsing.combinator.PackratParsers
-import scala.util.parsing.input.CharArrayReader.EofCh
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.expressions._
@@ -39,31 +35,7 @@ import org.apache.spark.sql.catalyst.types._
* This is currently included mostly for illustrative purposes. Users wanting more complete support
* for a SQL like language should checkout the HiveQL support in the sql/hive sub-project.
*/
-class SqlParser extends StandardTokenParsers with PackratParsers {
-
- def apply(input: String): LogicalPlan = {
- // Special-case out set commands since the value fields can be
- // complex to handle without RegexParsers. Also this approach
- // is clearer for the several possible cases of set commands.
- if (input.trim.toLowerCase.startsWith("set")) {
- input.trim.drop(3).split("=", 2).map(_.trim) match {
- case Array("") => // "set"
- SetCommand(None, None)
- case Array(key) => // "set key"
- SetCommand(Some(key), None)
- case Array(key, value) => // "set key=value"
- SetCommand(Some(key), Some(value))
- }
- } else {
- phrase(query)(new lexical.Scanner(input)) match {
- case Success(r, x) => r
- case x => sys.error(x.toString)
- }
- }
- }
-
- protected case class Keyword(str: String)
-
+class SqlParser extends AbstractSparkSQLParser {
protected implicit def asParser(k: Keyword): Parser[String] =
lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
@@ -100,7 +72,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers {
protected val IS = Keyword("IS")
protected val JOIN = Keyword("JOIN")
protected val LAST = Keyword("LAST")
- protected val LAZY = Keyword("LAZY")
protected val LEFT = Keyword("LEFT")
protected val LIKE = Keyword("LIKE")
protected val LIMIT = Keyword("LIMIT")
@@ -128,7 +99,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers {
protected val THEN = Keyword("THEN")
protected val TIMESTAMP = Keyword("TIMESTAMP")
protected val TRUE = Keyword("TRUE")
- protected val UNCACHE = Keyword("UNCACHE")
protected val UNION = Keyword("UNION")
protected val UPPER = Keyword("UPPER")
protected val WHEN = Keyword("WHEN")
@@ -136,7 +106,8 @@ class SqlParser extends StandardTokenParsers with PackratParsers {
// Use reflection to find the reserved words defined in this class.
protected val reservedWords =
- this.getClass
+ this
+ .getClass
.getMethods
.filter(_.getReturnType == classOf[Keyword])
.map(_.invoke(this).asInstanceOf[Keyword].str)
@@ -150,86 +121,68 @@ class SqlParser extends StandardTokenParsers with PackratParsers {
}
}
- protected lazy val query: Parser[LogicalPlan] = (
- select * (
- UNION ~ ALL ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Union(q1, q2) } |
- INTERSECT ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Intersect(q1, q2) } |
- EXCEPT ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Except(q1, q2)} |
- UNION ~ opt(DISTINCT) ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Distinct(Union(q1, q2)) }
+ protected lazy val start: Parser[LogicalPlan] =
+ ( select *
+ ( UNION ~ ALL ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Union(q1, q2) }
+ | INTERSECT ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Intersect(q1, q2) }
+ | EXCEPT ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Except(q1, q2)}
+ | UNION ~ DISTINCT.? ^^^ { (q1: LogicalPlan, q2: LogicalPlan) => Distinct(Union(q1, q2)) }
)
- | insert | cache | unCache
- )
+ | insert
+ )
protected lazy val select: Parser[LogicalPlan] =
- SELECT ~> opt(DISTINCT) ~ projections ~
- opt(from) ~ opt(filter) ~
- opt(grouping) ~
- opt(having) ~
- opt(orderBy) ~
- opt(limit) <~ opt(";") ^^ {
- case d ~ p ~ r ~ f ~ g ~ h ~ o ~ l =>
- val base = r.getOrElse(NoRelation)
- val withFilter = f.map(f => Filter(f, base)).getOrElse(base)
- val withProjection =
- g.map {g =>
- Aggregate(g, assignAliases(p), withFilter)
- }.getOrElse(Project(assignAliases(p), withFilter))
- val withDistinct = d.map(_ => Distinct(withProjection)).getOrElse(withProjection)
- val withHaving = h.map(h => Filter(h, withDistinct)).getOrElse(withDistinct)
- val withOrder = o.map(o => Sort(o, withHaving)).getOrElse(withHaving)
- val withLimit = l.map { l => Limit(l, withOrder) }.getOrElse(withOrder)
- withLimit
- }
+ SELECT ~> DISTINCT.? ~
+ repsep(projection, ",") ~
+ (FROM ~> relations).? ~
+ (WHERE ~> expression).? ~
+ (GROUP ~ BY ~> rep1sep(expression, ",")).? ~
+ (HAVING ~> expression).? ~
+ (ORDER ~ BY ~> ordering).? ~
+ (LIMIT ~> expression).? ^^ {
+ case d ~ p ~ r ~ f ~ g ~ h ~ o ~ l =>
+ val base = r.getOrElse(NoRelation)
+ val withFilter = f.map(f => Filter(f, base)).getOrElse(base)
+ val withProjection = g
+ .map(Aggregate(_, assignAliases(p), withFilter))
+ .getOrElse(Project(assignAliases(p), withFilter))
+ val withDistinct = d.map(_ => Distinct(withProjection)).getOrElse(withProjection)
+ val withHaving = h.map(Filter(_, withDistinct)).getOrElse(withDistinct)
+ val withOrder = o.map(Sort(_, withHaving)).getOrElse(withHaving)
+ val withLimit = l.map(Limit(_, withOrder)).getOrElse(withOrder)
+ withLimit
+ }
protected lazy val insert: Parser[LogicalPlan] =
- INSERT ~> opt(OVERWRITE) ~ inTo ~ select <~ opt(";") ^^ {
- case o ~ r ~ s =>
- val overwrite: Boolean = o.getOrElse("") == "OVERWRITE"
- InsertIntoTable(r, Map[String, Option[String]](), s, overwrite)
- }
-
- protected lazy val cache: Parser[LogicalPlan] =
- CACHE ~> opt(LAZY) ~ (TABLE ~> ident) ~ opt(AS ~> select) <~ opt(";") ^^ {
- case isLazy ~ tableName ~ plan =>
- CacheTableCommand(tableName, plan, isLazy.isDefined)
- }
-
- protected lazy val unCache: Parser[LogicalPlan] =
- UNCACHE ~ TABLE ~> ident <~ opt(";") ^^ {
- case tableName => UncacheTableCommand(tableName)
+ INSERT ~> OVERWRITE.? ~ (INTO ~> relation) ~ select ^^ {
+ case o ~ r ~ s => InsertIntoTable(r, Map.empty[String, Option[String]], s, o.isDefined)
}
- protected lazy val projections: Parser[Seq[Expression]] = repsep(projection, ",")
-
protected lazy val projection: Parser[Expression] =
- expression ~ (opt(AS) ~> opt(ident)) ^^ {
- case e ~ None => e
- case e ~ Some(a) => Alias(e, a)()
+ expression ~ (AS.? ~> ident.?) ^^ {
+ case e ~ a => a.fold(e)(Alias(e, _)())
}
- protected lazy val from: Parser[LogicalPlan] = FROM ~> relations
-
- protected lazy val inTo: Parser[LogicalPlan] = INTO ~> relation
-
// Based very loosely on the MySQL Grammar.
// http://dev.mysql.com/doc/refman/5.0/en/join.html
protected lazy val relations: Parser[LogicalPlan] =
- relation ~ "," ~ relation ^^ { case r1 ~ _ ~ r2 => Join(r1, r2, Inner, None) } |
- relation
+ ( relation ~ ("," ~> relation) ^^ { case r1 ~ r2 => Join(r1, r2, Inner, None) }
+ | relation
+ )
protected lazy val relation: Parser[LogicalPlan] =
- joinedRelation |
- relationFactor
+ joinedRelation | relationFactor
protected lazy val relationFactor: Parser[LogicalPlan] =
- ident ~ (opt(AS) ~> opt(ident)) ^^ {
- case tableName ~ alias => UnresolvedRelation(None, tableName, alias)
- } |
- "(" ~> query ~ ")" ~ opt(AS) ~ ident ^^ { case s ~ _ ~ _ ~ a => Subquery(a, s) }
+ ( ident ~ (opt(AS) ~> opt(ident)) ^^ {
+ case tableName ~ alias => UnresolvedRelation(None, tableName, alias)
+ }
+ | ("(" ~> start <~ ")") ~ (AS.? ~> ident) ^^ { case s ~ a => Subquery(a, s) }
+ )
protected lazy val joinedRelation: Parser[LogicalPlan] =
- relationFactor ~ opt(joinType) ~ JOIN ~ relationFactor ~ opt(joinConditions) ^^ {
- case r1 ~ jt ~ _ ~ r2 ~ cond =>
+ relationFactor ~ joinType.? ~ (JOIN ~> relationFactor) ~ joinConditions.? ^^ {
+ case r1 ~ jt ~ r2 ~ cond =>
Join(r1, r2, joinType = jt.getOrElse(Inner), cond)
}
@@ -237,160 +190,145 @@ class SqlParser extends StandardTokenParsers with PackratParsers {
ON ~> expression
protected lazy val joinType: Parser[JoinType] =
- INNER ^^^ Inner |
- LEFT ~ SEMI ^^^ LeftSemi |
- LEFT ~ opt(OUTER) ^^^ LeftOuter |
- RIGHT ~ opt(OUTER) ^^^ RightOuter |
- FULL ~ opt(OUTER) ^^^ FullOuter
-
- protected lazy val filter: Parser[Expression] = WHERE ~ expression ^^ { case _ ~ e => e }
-
- protected lazy val orderBy: Parser[Seq[SortOrder]] =
- ORDER ~> BY ~> ordering
+ ( INNER ^^^ Inner
+ | LEFT ~ SEMI ^^^ LeftSemi
+ | LEFT ~ OUTER.? ^^^ LeftOuter
+ | RIGHT ~ OUTER.? ^^^ RightOuter
+ | FULL ~ OUTER.? ^^^ FullOuter
+ )
protected lazy val ordering: Parser[Seq[SortOrder]] =
- rep1sep(singleOrder, ",") |
- rep1sep(expression, ",") ~ opt(direction) ^^ {
- case exps ~ None => exps.map(SortOrder(_, Ascending))
- case exps ~ Some(d) => exps.map(SortOrder(_, d))
- }
+ ( rep1sep(singleOrder, ",")
+ | rep1sep(expression, ",") ~ direction.? ^^ {
+ case exps ~ d => exps.map(SortOrder(_, d.getOrElse(Ascending)))
+ }
+ )
protected lazy val singleOrder: Parser[SortOrder] =
- expression ~ direction ^^ { case e ~ o => SortOrder(e,o) }
+ expression ~ direction ^^ { case e ~ o => SortOrder(e, o) }
protected lazy val direction: Parser[SortDirection] =
- ASC ^^^ Ascending |
- DESC ^^^ Descending
-
- protected lazy val grouping: Parser[Seq[Expression]] =
- GROUP ~> BY ~> rep1sep(expression, ",")
-
- protected lazy val having: Parser[Expression] =
- HAVING ~> expression
-
- protected lazy val limit: Parser[Expression] =
- LIMIT ~> expression
+ ( ASC ^^^ Ascending
+ | DESC ^^^ Descending
+ )
- protected lazy val expression: Parser[Expression] = orExpression
+ protected lazy val expression: Parser[Expression] =
+ orExpression
protected lazy val orExpression: Parser[Expression] =
- andExpression * (OR ^^^ { (e1: Expression, e2: Expression) => Or(e1,e2) })
+ andExpression * (OR ^^^ { (e1: Expression, e2: Expression) => Or(e1, e2) })
protected lazy val andExpression: Parser[Expression] =
- comparisonExpression * (AND ^^^ { (e1: Expression, e2: Expression) => And(e1,e2) })
+ comparisonExpression * (AND ^^^ { (e1: Expression, e2: Expression) => And(e1, e2) })
protected lazy val comparisonExpression: Parser[Expression] =
- termExpression ~ "=" ~ termExpression ^^ { case e1 ~ _ ~ e2 => EqualTo(e1, e2) } |
- termExpression ~ "<" ~ termExpression ^^ { case e1 ~ _ ~ e2 => LessThan(e1, e2) } |
- termExpression ~ "<=" ~ termExpression ^^ { case e1 ~ _ ~ e2 => LessThanOrEqual(e1, e2) } |
- termExpression ~ ">" ~ termExpression ^^ { case e1 ~ _ ~ e2 => GreaterThan(e1, e2) } |
- termExpression ~ ">=" ~ termExpression ^^ { case e1 ~ _ ~ e2 => GreaterThanOrEqual(e1, e2) } |
- termExpression ~ "!=" ~ termExpression ^^ { case e1 ~ _ ~ e2 => Not(EqualTo(e1, e2)) } |
- termExpression ~ "<>" ~ termExpression ^^ { case e1 ~ _ ~ e2 => Not(EqualTo(e1, e2)) } |
- termExpression ~ BETWEEN ~ termExpression ~ AND ~ termExpression ^^ {
- case e ~ _ ~ el ~ _ ~ eu => And(GreaterThanOrEqual(e, el), LessThanOrEqual(e, eu))
- } |
- termExpression ~ RLIKE ~ termExpression ^^ { case e1 ~ _ ~ e2 => RLike(e1, e2) } |
- termExpression ~ REGEXP ~ termExpression ^^ { case e1 ~ _ ~ e2 => RLike(e1, e2) } |
- termExpression ~ LIKE ~ termExpression ^^ { case e1 ~ _ ~ e2 => Like(e1, e2) } |
- termExpression ~ IN ~ "(" ~ rep1sep(termExpression, ",") <~ ")" ^^ {
- case e1 ~ _ ~ _ ~ e2 => In(e1, e2)
- } |
- termExpression ~ NOT ~ IN ~ "(" ~ rep1sep(termExpression, ",") <~ ")" ^^ {
- case e1 ~ _ ~ _ ~ _ ~ e2 => Not(In(e1, e2))
- } |
- termExpression <~ IS ~ NULL ^^ { case e => IsNull(e) } |
- termExpression <~ IS ~ NOT ~ NULL ^^ { case e => IsNotNull(e) } |
- NOT ~> termExpression ^^ {e => Not(e)} |
- termExpression
+ ( termExpression ~ ("=" ~> termExpression) ^^ { case e1 ~ e2 => EqualTo(e1, e2) }
+ | termExpression ~ ("<" ~> termExpression) ^^ { case e1 ~ e2 => LessThan(e1, e2) }
+ | termExpression ~ ("<=" ~> termExpression) ^^ { case e1 ~ e2 => LessThanOrEqual(e1, e2) }
+ | termExpression ~ (">" ~> termExpression) ^^ { case e1 ~ e2 => GreaterThan(e1, e2) }
+ | termExpression ~ (">=" ~> termExpression) ^^ { case e1 ~ e2 => GreaterThanOrEqual(e1, e2) }
+ | termExpression ~ ("!=" ~> termExpression) ^^ { case e1 ~ e2 => Not(EqualTo(e1, e2)) }
+ | termExpression ~ ("<>" ~> termExpression) ^^ { case e1 ~ e2 => Not(EqualTo(e1, e2)) }
+ | termExpression ~ (BETWEEN ~> termExpression) ~ (AND ~> termExpression) ^^ {
+ case e ~ el ~ eu => And(GreaterThanOrEqual(e, el), LessThanOrEqual(e, eu))
+ }
+ | termExpression ~ (RLIKE ~> termExpression) ^^ { case e1 ~ e2 => RLike(e1, e2) }
+ | termExpression ~ (REGEXP ~> termExpression) ^^ { case e1 ~ e2 => RLike(e1, e2) }
+ | termExpression ~ (LIKE ~> termExpression) ^^ { case e1 ~ e2 => Like(e1, e2) }
+ | termExpression ~ (IN ~ "(" ~> rep1sep(termExpression, ",")) <~ ")" ^^ {
+ case e1 ~ e2 => In(e1, e2)
+ }
+ | termExpression ~ (NOT ~ IN ~ "(" ~> rep1sep(termExpression, ",")) <~ ")" ^^ {
+ case e1 ~ e2 => Not(In(e1, e2))
+ }
+ | termExpression <~ IS ~ NULL ^^ { case e => IsNull(e) }
+ | termExpression <~ IS ~ NOT ~ NULL ^^ { case e => IsNotNull(e) }
+ | NOT ~> termExpression ^^ {e => Not(e)}
+ | termExpression
+ )
protected lazy val termExpression: Parser[Expression] =
- productExpression * (
- "+" ^^^ { (e1: Expression, e2: Expression) => Add(e1,e2) } |
- "-" ^^^ { (e1: Expression, e2: Expression) => Subtract(e1,e2) } )
+ productExpression *
+ ( "+" ^^^ { (e1: Expression, e2: Expression) => Add(e1, e2) }
+ | "-" ^^^ { (e1: Expression, e2: Expression) => Subtract(e1, e2) }
+ )
protected lazy val productExpression: Parser[Expression] =
- baseExpression * (
- "*" ^^^ { (e1: Expression, e2: Expression) => Multiply(e1,e2) } |
- "/" ^^^ { (e1: Expression, e2: Expression) => Divide(e1,e2) } |
- "%" ^^^ { (e1: Expression, e2: Expression) => Remainder(e1,e2) }
- )
+ baseExpression *
+ ( "*" ^^^ { (e1: Expression, e2: Expression) => Multiply(e1, e2) }
+ | "/" ^^^ { (e1: Expression, e2: Expression) => Divide(e1, e2) }
+ | "%" ^^^ { (e1: Expression, e2: Expression) => Remainder(e1, e2) }
+ )
protected lazy val function: Parser[Expression] =
- SUM ~> "(" ~> expression <~ ")" ^^ { case exp => Sum(exp) } |
- SUM ~> "(" ~> DISTINCT ~> expression <~ ")" ^^ { case exp => SumDistinct(exp) } |
- COUNT ~> "(" ~ "*" <~ ")" ^^ { case _ => Count(Literal(1)) } |
- COUNT ~> "(" ~ expression <~ ")" ^^ { case dist ~ exp => Count(exp) } |
- COUNT ~> "(" ~> DISTINCT ~> expression <~ ")" ^^ { case exp => CountDistinct(exp :: Nil) } |
- APPROXIMATE ~> COUNT ~> "(" ~> DISTINCT ~> expression <~ ")" ^^ {
- case exp => ApproxCountDistinct(exp)
- } |
- APPROXIMATE ~> "(" ~> floatLit ~ ")" ~ COUNT ~ "(" ~ DISTINCT ~ expression <~ ")" ^^ {
- case s ~ _ ~ _ ~ _ ~ _ ~ e => ApproxCountDistinct(e, s.toDouble)
- } |
- FIRST ~> "(" ~> expression <~ ")" ^^ { case exp => First(exp) } |
- LAST ~> "(" ~> expression <~ ")" ^^ { case exp => Last(exp) } |
- AVG ~> "(" ~> expression <~ ")" ^^ { case exp => Average(exp) } |
- MIN ~> "(" ~> expression <~ ")" ^^ { case exp => Min(exp) } |
- MAX ~> "(" ~> expression <~ ")" ^^ { case exp => Max(exp) } |
- UPPER ~> "(" ~> expression <~ ")" ^^ { case exp => Upper(exp) } |
- LOWER ~> "(" ~> expression <~ ")" ^^ { case exp => Lower(exp) } |
- IF ~> "(" ~> expression ~ "," ~ expression ~ "," ~ expression <~ ")" ^^ {
- case c ~ "," ~ t ~ "," ~ f => If(c,t,f)
- } |
- CASE ~> expression.? ~ (WHEN ~> expression ~ (THEN ~> expression)).* ~
- (ELSE ~> expression).? <~ END ^^ {
- case casePart ~ altPart ~ elsePart =>
- val altExprs = altPart.flatMap {
- case we ~ te =>
- Seq(casePart.fold(we)(EqualTo(_, we)), te)
+ ( SUM ~> "(" ~> expression <~ ")" ^^ { case exp => Sum(exp) }
+ | SUM ~> "(" ~> DISTINCT ~> expression <~ ")" ^^ { case exp => SumDistinct(exp) }
+ | COUNT ~ "(" ~> "*" <~ ")" ^^ { case _ => Count(Literal(1)) }
+ | COUNT ~ "(" ~> expression <~ ")" ^^ { case exp => Count(exp) }
+ | COUNT ~> "(" ~> DISTINCT ~> expression <~ ")" ^^ { case exp => CountDistinct(exp :: Nil) }
+ | APPROXIMATE ~ COUNT ~ "(" ~ DISTINCT ~> expression <~ ")" ^^
+ { case exp => ApproxCountDistinct(exp) }
+ | APPROXIMATE ~> "(" ~> floatLit ~ ")" ~ COUNT ~ "(" ~ DISTINCT ~ expression <~ ")" ^^
+ { case s ~ _ ~ _ ~ _ ~ _ ~ e => ApproxCountDistinct(e, s.toDouble) }
+ | FIRST ~ "(" ~> expression <~ ")" ^^ { case exp => First(exp) }
+ | LAST ~ "(" ~> expression <~ ")" ^^ { case exp => Last(exp) }
+ | AVG ~ "(" ~> expression <~ ")" ^^ { case exp => Average(exp) }
+ | MIN ~ "(" ~> expression <~ ")" ^^ { case exp => Min(exp) }
+ | MAX ~ "(" ~> expression <~ ")" ^^ { case exp => Max(exp) }
+ | UPPER ~ "(" ~> expression <~ ")" ^^ { case exp => Upper(exp) }
+ | LOWER ~ "(" ~> expression <~ ")" ^^ { case exp => Lower(exp) }
+ | IF ~ "(" ~> expression ~ ("," ~> expression) ~ ("," ~> expression) <~ ")" ^^
+ { case c ~ t ~ f => If(c, t, f) }
+ | CASE ~> expression.? ~ (WHEN ~> expression ~ (THEN ~> expression)).* ~
+ (ELSE ~> expression).? <~ END ^^ {
+ case casePart ~ altPart ~ elsePart =>
+ val altExprs = altPart.flatMap { case whenExpr ~ thenExpr =>
+ Seq(casePart.fold(whenExpr)(EqualTo(_, whenExpr)), thenExpr)
+ }
+ CaseWhen(altExprs ++ elsePart.toList)
}
- CaseWhen(altExprs ++ elsePart.toList)
- } |
- (SUBSTR | SUBSTRING) ~> "(" ~> expression ~ "," ~ expression <~ ")" ^^ {
- case s ~ "," ~ p => Substring(s,p,Literal(Integer.MAX_VALUE))
- } |
- (SUBSTR | SUBSTRING) ~> "(" ~> expression ~ "," ~ expression ~ "," ~ expression <~ ")" ^^ {
- case s ~ "," ~ p ~ "," ~ l => Substring(s,p,l)
- } |
- SQRT ~> "(" ~> expression <~ ")" ^^ { case exp => Sqrt(exp) } |
- ABS ~> "(" ~> expression <~ ")" ^^ { case exp => Abs(exp) } |
- ident ~ "(" ~ repsep(expression, ",") <~ ")" ^^ {
- case udfName ~ _ ~ exprs => UnresolvedFunction(udfName, exprs)
- }
+ | (SUBSTR | SUBSTRING) ~ "(" ~> expression ~ ("," ~> expression) <~ ")" ^^
+ { case s ~ p => Substring(s, p, Literal(Integer.MAX_VALUE)) }
+ | (SUBSTR | SUBSTRING) ~ "(" ~> expression ~ ("," ~> expression) ~ ("," ~> expression) <~ ")" ^^
+ { case s ~ p ~ l => Substring(s, p, l) }
+ | SQRT ~ "(" ~> expression <~ ")" ^^ { case exp => Sqrt(exp) }
+ | ABS ~ "(" ~> expression <~ ")" ^^ { case exp => Abs(exp) }
+ | ident ~ ("(" ~> repsep(expression, ",")) <~ ")" ^^
+ { case udfName ~ exprs => UnresolvedFunction(udfName, exprs) }
+ )
protected lazy val cast: Parser[Expression] =
- CAST ~> "(" ~> expression ~ AS ~ dataType <~ ")" ^^ { case exp ~ _ ~ t => Cast(exp, t) }
+ CAST ~ "(" ~> expression ~ (AS ~> dataType) <~ ")" ^^ { case exp ~ t => Cast(exp, t) }
protected lazy val literal: Parser[Literal] =
- numericLit ^^ {
- case i if i.toLong > Int.MaxValue => Literal(i.toLong)
- case i => Literal(i.toInt)
- } |
- NULL ^^^ Literal(null, NullType) |
- floatLit ^^ {case f => Literal(f.toDouble) } |
- stringLit ^^ {case s => Literal(s, StringType) }
+ ( numericLit ^^ {
+ case i if i.toLong > Int.MaxValue => Literal(i.toLong)
+ case i => Literal(i.toInt)
+ }
+ | NULL ^^^ Literal(null, NullType)
+ | floatLit ^^ {case f => Literal(f.toDouble) }
+ | stringLit ^^ {case s => Literal(s, StringType) }
+ )
protected lazy val floatLit: Parser[String] =
elem("decimal", _.isInstanceOf[lexical.FloatLit]) ^^ (_.chars)
protected lazy val baseExpression: PackratParser[Expression] =
- expression ~ "[" ~ expression <~ "]" ^^ {
- case base ~ _ ~ ordinal => GetItem(base, ordinal)
- } |
- (expression <~ ".") ~ ident ^^ {
- case base ~ fieldName => GetField(base, fieldName)
- } |
- TRUE ^^^ Literal(true, BooleanType) |
- FALSE ^^^ Literal(false, BooleanType) |
- cast |
- "(" ~> expression <~ ")" |
- function |
- "-" ~> literal ^^ UnaryMinus |
- dotExpressionHeader |
- ident ^^ UnresolvedAttribute |
- "*" ^^^ Star(None) |
- literal
+ ( expression ~ ("[" ~> expression <~ "]") ^^
+ { case base ~ ordinal => GetItem(base, ordinal) }
+ | (expression <~ ".") ~ ident ^^
+ { case base ~ fieldName => GetField(base, fieldName) }
+ | TRUE ^^^ Literal(true, BooleanType)
+ | FALSE ^^^ Literal(false, BooleanType)
+ | cast
+ | "(" ~> expression <~ ")"
+ | function
+ | "-" ~> literal ^^ UnaryMinus
+ | dotExpressionHeader
+ | ident ^^ UnresolvedAttribute
+ | "*" ^^^ Star(None)
+ | literal
+ )
protected lazy val dotExpressionHeader: Parser[Expression] =
(ident <~ ".") ~ ident ~ rep("." ~> ident) ^^ {
@@ -400,55 +338,3 @@ class SqlParser extends StandardTokenParsers with PackratParsers {
protected lazy val dataType: Parser[DataType] =
STRING ^^^ StringType | TIMESTAMP ^^^ TimestampType
}
-
-class SqlLexical(val keywords: Seq[String]) extends StdLexical {
- case class FloatLit(chars: String) extends Token {
- override def toString = chars
- }
-
- reserved ++= keywords.flatMap(w => allCaseVersions(w))
-
- delimiters += (
- "@", "*", "+", "-", "<", "=", "<>", "!=", "<=", ">=", ">", "/", "(", ")",
- ",", ";", "%", "{", "}", ":", "[", "]", "."
- )
-
- override lazy val token: Parser[Token] = (
- identChar ~ rep( identChar | digit ) ^^
- { case first ~ rest => processIdent(first :: rest mkString "") }
- | rep1(digit) ~ opt('.' ~> rep(digit)) ^^ {
- case i ~ None => NumericLit(i mkString "")
- case i ~ Some(d) => FloatLit(i.mkString("") + "." + d.mkString(""))
- }
- | '\'' ~ rep( chrExcept('\'', '\n', EofCh) ) ~ '\'' ^^
- { case '\'' ~ chars ~ '\'' => StringLit(chars mkString "") }
- | '\"' ~ rep( chrExcept('\"', '\n', EofCh) ) ~ '\"' ^^
- { case '\"' ~ chars ~ '\"' => StringLit(chars mkString "") }
- | EofCh ^^^ EOF
- | '\'' ~> failure("unclosed string literal")
- | '\"' ~> failure("unclosed string literal")
- | delim
- | failure("illegal character")
- )
-
- override def identChar = letter | elem('_')
-
- override def whitespace: Parser[Any] = rep(
- whitespaceChar
- | '/' ~ '*' ~ comment
- | '/' ~ '/' ~ rep( chrExcept(EofCh, '\n') )
- | '#' ~ rep( chrExcept(EofCh, '\n') )
- | '-' ~ '-' ~ rep( chrExcept(EofCh, '\n') )
- | '/' ~ '*' ~ failure("unclosed comment")
- )
-
- /** Generate all variations of upper and lower case of a given string */
- def allCaseVersions(s: String, prefix: String = ""): Stream[String] = {
- if (s == "") {
- Stream(prefix)
- } else {
- allCaseVersions(s.tail, prefix + s.head.toLower) ++
- allCaseVersions(s.tail, prefix + s.head.toUpper)
- }
- }
-}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/commands.scala
index 9a3848cfc6b620b82521b5b80538b3184777093f..b8ba2ee428a20f68b9f2d307823d722f2680801b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/commands.scala
@@ -39,9 +39,9 @@ case class NativeCommand(cmd: String) extends Command {
}
/**
- * Commands of the form "SET (key) (= value)".
+ * Commands of the form "SET [key [= value] ]".
*/
-case class SetCommand(key: Option[String], value: Option[String]) extends Command {
+case class SetCommand(kv: Option[(String, Option[String])]) extends Command {
override def output = Seq(
AttributeReference("", StringType, nullable = false)())
}
@@ -81,3 +81,14 @@ case class DescribeCommand(
AttributeReference("data_type", StringType, nullable = false)(),
AttributeReference("comment", StringType, nullable = false)())
}
+
+/**
+ * Returned for the "! shellCommand" command
+ */
+case class ShellCommand(cmd: String) extends Command
+
+
+/**
+ * Returned for the "SOURCE file" command
+ */
+case class SourceCommand(filePath: String) extends Command
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 014e1e2826724906178fa7f0d40d292f7342e5c6..23e7b2d27077721bd0d70e9b9cd83faa1e322c00 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -66,12 +66,17 @@ class SQLContext(@transient val sparkContext: SparkContext)
@transient
protected[sql] lazy val analyzer: Analyzer =
new Analyzer(catalog, functionRegistry, caseSensitive = true)
+
@transient
protected[sql] val optimizer = Optimizer
+
@transient
- protected[sql] val parser = new catalyst.SqlParser
+ protected[sql] val sqlParser = {
+ val fallback = new catalyst.SqlParser
+ new catalyst.SparkSQLParser(fallback(_))
+ }
- protected[sql] def parseSql(sql: String): LogicalPlan = parser(sql)
+ protected[sql] def parseSql(sql: String): LogicalPlan = sqlParser(sql)
protected[sql] def executeSql(sql: String): this.QueryExecution = executePlan(parseSql(sql))
protected[sql] def executePlan(plan: LogicalPlan): this.QueryExecution =
new this.QueryExecution { val logical = plan }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index bbf17b9fadf860b2602e51a463594e617e9a6830..4f1af7234d551ce2c2b2417cc5e801b01ac31500 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -304,8 +304,8 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
case class CommandStrategy(context: SQLContext) extends Strategy {
def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
- case logical.SetCommand(key, value) =>
- Seq(execution.SetCommand(key, value, plan.output)(context))
+ case logical.SetCommand(kv) =>
+ Seq(execution.SetCommand(kv, plan.output)(context))
case logical.ExplainCommand(logicalPlan, extended) =>
Seq(execution.ExplainCommand(logicalPlan, plan.output, extended)(context))
case logical.CacheTableCommand(tableName, optPlan, isLazy) =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala
index d49633c24ad4dae7e68b28211d2f4f8191546329..5859eba408ee1dfdcea836540c9f5195fd749165 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala
@@ -48,29 +48,28 @@ trait Command {
* :: DeveloperApi ::
*/
@DeveloperApi
-case class SetCommand(
- key: Option[String], value: Option[String], output: Seq[Attribute])(
+case class SetCommand(kv: Option[(String, Option[String])], output: Seq[Attribute])(
@transient context: SQLContext)
extends LeafNode with Command with Logging {
- override protected lazy val sideEffectResult: Seq[Row] = (key, value) match {
- // Set value for key k.
- case (Some(k), Some(v)) =>
- if (k == SQLConf.Deprecated.MAPRED_REDUCE_TASKS) {
+ override protected lazy val sideEffectResult: Seq[Row] = kv match {
+ // Set value for the key.
+ case Some((key, Some(value))) =>
+ if (key == SQLConf.Deprecated.MAPRED_REDUCE_TASKS) {
logWarning(s"Property ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} is deprecated, " +
s"automatically converted to ${SQLConf.SHUFFLE_PARTITIONS} instead.")
- context.setConf(SQLConf.SHUFFLE_PARTITIONS, v)
- Seq(Row(s"${SQLConf.SHUFFLE_PARTITIONS}=$v"))
+ context.setConf(SQLConf.SHUFFLE_PARTITIONS, value)
+ Seq(Row(s"${SQLConf.SHUFFLE_PARTITIONS}=$value"))
} else {
- context.setConf(k, v)
- Seq(Row(s"$k=$v"))
+ context.setConf(key, value)
+ Seq(Row(s"$key=$value"))
}
- // Query the value bound to key k.
- case (Some(k), _) =>
+ // Query the value bound to the key.
+ case Some((key, None)) =>
// TODO (lian) This is just a workaround to make the Simba ODBC driver work.
// Should remove this once we get the ODBC driver updated.
- if (k == "-v") {
+ if (key == "-v") {
val hiveJars = Seq(
"hive-exec-0.12.0.jar",
"hive-service-0.12.0.jar",
@@ -84,23 +83,20 @@ case class SetCommand(
Row("system:java.class.path=" + hiveJars),
Row("system:sun.java.command=shark.SharkServer2"))
} else {
- if (k == SQLConf.Deprecated.MAPRED_REDUCE_TASKS) {
+ if (key == SQLConf.Deprecated.MAPRED_REDUCE_TASKS) {
logWarning(s"Property ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} is deprecated, " +
s"showing ${SQLConf.SHUFFLE_PARTITIONS} instead.")
Seq(Row(s"${SQLConf.SHUFFLE_PARTITIONS}=${context.numShufflePartitions}"))
} else {
- Seq(Row(s"$k=${context.getConf(k, "<undefined>")}"))
+ Seq(Row(s"$key=${context.getConf(key, "<undefined>")}"))
}
}
// Query all key-value pairs that are set in the SQLConf of the context.
- case (None, None) =>
+ case _ =>
context.getAllConfs.map { case (k, v) =>
Row(s"$k=$v")
}.toSeq
-
- case _ =>
- throw new IllegalArgumentException()
}
override def otherCopyArgs = context :: Nil
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index 1e624f97004f56183d509fa797ad4f9b46fafc4c..c87ded81fdc2786ded030ff8e9dd5ec7e573b268 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -69,7 +69,7 @@ class CachedTableSuite extends QueryTest {
test("calling .unpersist() should drop in-memory columnar cache") {
table("testData").cache()
table("testData").count()
- table("testData").unpersist(true)
+ table("testData").unpersist(blocking = true)
assertCached(table("testData"), 0)
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 79de1bb855dbe9ed809885ad6e9b16bf65e49e6f..a94022c0cf6e369b90a1c8f6ebcd49579ff4213f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -42,7 +42,6 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
TimeZone.setDefault(origZone)
}
-
test("SPARK-3176 Added Parser of SQL ABS()") {
checkAnswer(
sql("SELECT ABS(-1.3)"),
@@ -61,7 +60,6 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
4)
}
-
test("SPARK-2041 column name equals tablename") {
checkAnswer(
sql("SELECT tableName FROM tableName"),
@@ -694,6 +692,6 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
test("SPARK-3813 CASE WHEN a THEN b [WHEN c THEN d]* [ELSE e] END") {
checkAnswer(
- sql("SELECT CASE WHEN key=1 THEN 1 ELSE 2 END FROM testData WHERE key = 1 group by key"), 1)
+ sql("SELECT CASE WHEN key = 1 THEN 1 ELSE 2 END FROM testData WHERE key = 1 group by key"), 1)
}
}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
index 910174a1537682531f9e62dc48bada8289717b6f..accf61576b8042a2f5db320c2598656cda15c5d6 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
@@ -172,7 +172,7 @@ private[thriftserver] class SparkSQLOperationManager(hiveContext: HiveContext)
result = hiveContext.sql(statement)
logDebug(result.queryExecution.toString())
result.queryExecution.logical match {
- case SetCommand(Some(key), Some(value)) if (key == SQLConf.THRIFTSERVER_POOL) =>
+ case SetCommand(Some((SQLConf.THRIFTSERVER_POOL, Some(value)))) =>
sessionToActivePool(parentSession) = value
logInfo(s"Setting spark.scheduler.pool=$value for future statements in this session.")
case _ =>
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala
index c5844e92eaaa9d6af6edc48db544e90cfeab566b..430ffb29989eadbbf981b3a530861036e82afb9a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/ExtendedHiveQlParser.scala
@@ -18,118 +18,50 @@
package org.apache.spark.sql.hive
import scala.language.implicitConversions
-import scala.util.parsing.combinator.syntactical.StandardTokenParsers
-import scala.util.parsing.combinator.PackratParsers
+
import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.SqlLexical
+import org.apache.spark.sql.catalyst.{AbstractSparkSQLParser, SqlLexical}
/**
- * A parser that recognizes all HiveQL constructs together with several Spark SQL specific
- * extensions like CACHE TABLE and UNCACHE TABLE.
+ * A parser that recognizes all HiveQL constructs together with Spark SQL specific extensions.
*/
-private[hive] class ExtendedHiveQlParser extends StandardTokenParsers with PackratParsers {
-
- def apply(input: String): LogicalPlan = {
- // Special-case out set commands since the value fields can be
- // complex to handle without RegexParsers. Also this approach
- // is clearer for the several possible cases of set commands.
- if (input.trim.toLowerCase.startsWith("set")) {
- input.trim.drop(3).split("=", 2).map(_.trim) match {
- case Array("") => // "set"
- SetCommand(None, None)
- case Array(key) => // "set key"
- SetCommand(Some(key), None)
- case Array(key, value) => // "set key=value"
- SetCommand(Some(key), Some(value))
- }
- } else if (input.trim.startsWith("!")) {
- ShellCommand(input.drop(1))
- } else {
- phrase(query)(new lexical.Scanner(input)) match {
- case Success(r, x) => r
- case x => sys.error(x.toString)
- }
- }
- }
-
- protected case class Keyword(str: String)
-
- protected val ADD = Keyword("ADD")
- protected val AS = Keyword("AS")
- protected val CACHE = Keyword("CACHE")
- protected val DFS = Keyword("DFS")
- protected val FILE = Keyword("FILE")
- protected val JAR = Keyword("JAR")
- protected val LAZY = Keyword("LAZY")
- protected val SET = Keyword("SET")
- protected val SOURCE = Keyword("SOURCE")
- protected val TABLE = Keyword("TABLE")
- protected val UNCACHE = Keyword("UNCACHE")
-
+private[hive] class ExtendedHiveQlParser extends AbstractSparkSQLParser {
protected implicit def asParser(k: Keyword): Parser[String] =
lexical.allCaseVersions(k.str).map(x => x : Parser[String]).reduce(_ | _)
- protected def allCaseConverse(k: String): Parser[String] =
- lexical.allCaseVersions(k).map(x => x : Parser[String]).reduce(_ | _)
+ protected val ADD = Keyword("ADD")
+ protected val DFS = Keyword("DFS")
+ protected val FILE = Keyword("FILE")
+ protected val JAR = Keyword("JAR")
- protected val reservedWords =
- this.getClass
+ private val reservedWords =
+ this
+ .getClass
.getMethods
.filter(_.getReturnType == classOf[Keyword])
.map(_.invoke(this).asInstanceOf[Keyword].str)
override val lexical = new SqlLexical(reservedWords)
- protected lazy val query: Parser[LogicalPlan] =
- cache | uncache | addJar | addFile | dfs | source | hiveQl
+ protected lazy val start: Parser[LogicalPlan] = dfs | addJar | addFile | hiveQl
protected lazy val hiveQl: Parser[LogicalPlan] =
restInput ^^ {
- case statement => HiveQl.createPlan(statement.trim())
- }
-
- // Returns the whole input string
- protected lazy val wholeInput: Parser[String] = new Parser[String] {
- def apply(in: Input) =
- Success(in.source.toString, in.drop(in.source.length()))
- }
-
- // Returns the rest of the input string that are not parsed yet
- protected lazy val restInput: Parser[String] = new Parser[String] {
- def apply(in: Input) =
- Success(
- in.source.subSequence(in.offset, in.source.length).toString,
- in.drop(in.source.length()))
- }
-
- protected lazy val cache: Parser[LogicalPlan] =
- CACHE ~> opt(LAZY) ~ (TABLE ~> ident) ~ opt(AS ~> hiveQl) ^^ {
- case isLazy ~ tableName ~ plan =>
- CacheTableCommand(tableName, plan, isLazy.isDefined)
- }
-
- protected lazy val uncache: Parser[LogicalPlan] =
- UNCACHE ~ TABLE ~> ident ^^ {
- case tableName => UncacheTableCommand(tableName)
+ case statement => HiveQl.createPlan(statement.trim)
}
- protected lazy val addJar: Parser[LogicalPlan] =
- ADD ~ JAR ~> restInput ^^ {
- case jar => AddJar(jar.trim())
+ protected lazy val dfs: Parser[LogicalPlan] =
+ DFS ~> wholeInput ^^ {
+ case command => NativeCommand(command.trim)
}
- protected lazy val addFile: Parser[LogicalPlan] =
+ private lazy val addFile: Parser[LogicalPlan] =
ADD ~ FILE ~> restInput ^^ {
- case file => AddFile(file.trim())
+ case input => AddFile(input.trim)
}
- protected lazy val dfs: Parser[LogicalPlan] =
- DFS ~> wholeInput ^^ {
- case command => NativeCommand(command.trim())
- }
-
- protected lazy val source: Parser[LogicalPlan] =
- SOURCE ~> restInput ^^ {
- case file => SourceCommand(file.trim())
+ private lazy val addJar: Parser[LogicalPlan] =
+ ADD ~ JAR ~> restInput ^^ {
+ case input => AddJar(input.trim)
}
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
index 32c9175f181bbd27d943cbd66ff2c01526e91583..98a46a31e1ffd7949ca364d38c3730b1f209c92d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
@@ -21,6 +21,7 @@ import org.apache.hadoop.hive.ql.lib.Node
import org.apache.hadoop.hive.ql.parse._
import org.apache.hadoop.hive.ql.plan.PlanUtils
+import org.apache.spark.sql.catalyst.SparkSQLParser
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans._
@@ -38,10 +39,6 @@ import scala.collection.JavaConversions._
*/
private[hive] case object NativePlaceholder extends Command
-private[hive] case class ShellCommand(cmd: String) extends Command
-
-private[hive] case class SourceCommand(filePath: String) extends Command
-
private[hive] case class AddFile(filePath: String) extends Command
private[hive] case class AddJar(path: String) extends Command
@@ -126,9 +123,11 @@ private[hive] object HiveQl {
"TOK_CREATETABLE",
"TOK_DESCTABLE"
) ++ nativeCommands
-
- // It parses hive sql query along with with several Spark SQL specific extensions
- protected val hiveSqlParser = new ExtendedHiveQlParser
+
+ protected val hqlParser = {
+ val fallback = new ExtendedHiveQlParser
+ new SparkSQLParser(fallback(_))
+ }
/**
* A set of implicit transformations that allow Hive ASTNodes to be rewritten by transformations
@@ -218,7 +217,7 @@ private[hive] object HiveQl {
def getAst(sql: String): ASTNode = ParseUtils.findRootNonNullToken((new ParseDriver).parse(sql))
/** Returns a LogicalPlan for a given HiveQL string. */
- def parseSql(sql: String): LogicalPlan = hiveSqlParser(sql)
+ def parseSql(sql: String): LogicalPlan = hqlParser(sql)
/** Creates LogicalPlan for a given HiveQL string. */
def createPlan(sql: String) = {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
index 508d8239c762834837d27bb5a7dcd6d52dc49320..5c66322f1ed997454af040f43f067e3c91790611 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala
@@ -167,10 +167,10 @@ private[hive] trait HiveStrategies {
database.get,
tableName,
query,
- InsertIntoHiveTable(_: MetastoreRelation,
- Map(),
- query,
- true)(hiveContext)) :: Nil
+ InsertIntoHiveTable(_: MetastoreRelation,
+ Map(),
+ query,
+ overwrite = true)(hiveContext)) :: Nil
case _ => Nil
}
}