Skip to content
Snippets Groups Projects
Commit 6325a2f8 authored by Takeshi Yamamuro's avatar Takeshi Yamamuro Committed by Herman van Hovell
Browse files

[SPARK-19923][SQL] Remove unnecessary type conversions per call in Hive

## What changes were proposed in this pull request?
This pr removed unnecessary type conversions per call in Hive: https://github.com/apache/spark/blob/master/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala#L116

## How was this patch tested?
Existing tests

Author: Takeshi Yamamuro <yamamuro@apache.org>

Closes #17264 from maropu/SPARK-19923.
parent a02a0b17
No related branches found
No related tags found
No related merge requests found
......@@ -108,12 +108,13 @@ private[hive] case class HiveSimpleUDF(
private[hive] class DeferredObjectAdapter(oi: ObjectInspector, dataType: DataType)
extends DeferredObject with HiveInspectors {
private val wrapper = wrapperFor(oi, dataType)
private var func: () => Any = _
def set(func: () => Any): Unit = {
this.func = func
}
override def prepare(i: Int): Unit = {}
override def get(): AnyRef = wrap(func(), oi, dataType)
override def get(): AnyRef = wrapper(func()).asInstanceOf[AnyRef]
}
private[hive] case class HiveGenericUDF(
......
......@@ -20,6 +20,8 @@ package org.apache.spark.sql.hive.orc
import java.net.URI
import java.util.Properties
import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileStatus, Path}
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
......@@ -196,6 +198,11 @@ private[orc] class OrcSerializer(dataSchema: StructType, conf: Configuration)
private[this] val cachedOrcStruct = structOI.create().asInstanceOf[OrcStruct]
// Wrapper functions used to wrap Spark SQL input arguments into Hive specific format
private[this] val wrappers = dataSchema.zip(structOI.getAllStructFieldRefs().asScala.toSeq).map {
case (f, i) => wrapperFor(i.getFieldObjectInspector, f.dataType)
}
private[this] def wrapOrcStruct(
struct: OrcStruct,
oi: SettableStructObjectInspector,
......@@ -208,10 +215,8 @@ private[orc] class OrcSerializer(dataSchema: StructType, conf: Configuration)
oi.setStructFieldData(
struct,
fieldRefs.get(i),
wrap(
row.get(i, dataSchema(i).dataType),
fieldRefs.get(i).getFieldObjectInspector,
dataSchema(i).dataType))
wrappers(i)(row.get(i, dataSchema(i).dataType))
)
i += 1
}
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment