Skip to content
Snippets Groups Projects
Commit 3a652f69 authored by Reynold Xin's avatar Reynold Xin
Browse files

[SPARK-11561][SQL] Rename text data source's column name to value.

Author: Reynold Xin <rxin@databricks.com>

Closes #9527 from rxin/SPARK-11561.
parent f328feda
No related branches found
No related tags found
No related merge requests found
...@@ -30,14 +30,12 @@ import org.apache.spark.deploy.SparkHadoopUtil ...@@ -30,14 +30,12 @@ import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.mapred.SparkHadoopMapRedUtil import org.apache.spark.mapred.SparkHadoopMapRedUtil
import org.apache.spark.rdd.RDD import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{UnsafeRow, GenericMutableRow} import org.apache.spark.sql.catalyst.expressions.UnsafeRow
import org.apache.spark.sql.catalyst.expressions.codegen.{UnsafeRowWriter, BufferHolder} import org.apache.spark.sql.catalyst.expressions.codegen.{UnsafeRowWriter, BufferHolder}
import org.apache.spark.sql.columnar.MutableUnsafeRow
import org.apache.spark.sql.{AnalysisException, Row, SQLContext} import org.apache.spark.sql.{AnalysisException, Row, SQLContext}
import org.apache.spark.sql.execution.datasources.PartitionSpec import org.apache.spark.sql.execution.datasources.PartitionSpec
import org.apache.spark.sql.sources._ import org.apache.spark.sql.sources._
import org.apache.spark.sql.types.{StringType, StructType} import org.apache.spark.sql.types.{StringType, StructType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.SerializableConfiguration import org.apache.spark.util.SerializableConfiguration
/** /**
...@@ -78,7 +76,7 @@ private[sql] class TextRelation( ...@@ -78,7 +76,7 @@ private[sql] class TextRelation(
extends HadoopFsRelation(maybePartitionSpec) { extends HadoopFsRelation(maybePartitionSpec) {
/** Data schema is always a single column, named "text". */ /** Data schema is always a single column, named "text". */
override def dataSchema: StructType = new StructType().add("text", StringType) override def dataSchema: StructType = new StructType().add("value", StringType)
/** This is an internal data source that outputs internal row format. */ /** This is an internal data source that outputs internal row format. */
override val needConversion: Boolean = false override val needConversion: Boolean = false
......
...@@ -65,7 +65,7 @@ class TextSuite extends QueryTest with SharedSQLContext { ...@@ -65,7 +65,7 @@ class TextSuite extends QueryTest with SharedSQLContext {
/** Verifies data and schema. */ /** Verifies data and schema. */
private def verifyFrame(df: DataFrame): Unit = { private def verifyFrame(df: DataFrame): Unit = {
// schema // schema
assert(df.schema == new StructType().add("text", StringType)) assert(df.schema == new StructType().add("value", StringType))
// verify content // verify content
val data = df.collect() val data = df.collect()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment