From 2ca563cc45d1ac1c19b8e84c5a87a950c712ab87 Mon Sep 17 00:00:00 2001
From: Lianhui Wang <lianhuiwang09@gmail.com>
Date: Fri, 3 Jun 2016 22:19:22 -0700
Subject: [PATCH] [SPARK-15756][SQL] Support command 'create table stored as
orcfile/parquetfile/avrofile'
## What changes were proposed in this pull request?
Now Spark SQL can support 'create table src stored as orc/parquet/avro' for orc/parquet/avro table. But Hive can support both commands: ' stored as orc/parquet/avro' and 'stored as orcfile/parquetfile/avrofile'.
So this PR supports these keywords 'orcfile/parquetfile/avrofile' in Spark SQL.
## How was this patch tested?
add unit tests
Author: Lianhui Wang <lianhuiwang09@gmail.com>
Closes #13500 from lianhuiwang/SPARK-15756.
---
.../org/apache/spark/sql/internal/HiveSerDe.scala | 3 +++
.../sql/execution/command/DDLCommandSuite.scala | 15 +++++++++++++++
2 files changed, 18 insertions(+)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
index d554937d8b..ad69137f74 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
@@ -71,6 +71,9 @@ object HiveSerDe {
val key = source.toLowerCase match {
case s if s.startsWith("org.apache.spark.sql.parquet") => "parquet"
case s if s.startsWith("org.apache.spark.sql.orc") => "orc"
+ case s if s.equals("orcfile") => "orc"
+ case s if s.equals("parquetfile") => "parquet"
+ case s if s.equals("avrofile") => "avro"
case s => s
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala
index 850fca5852..aec7e99d9d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala
@@ -237,6 +237,21 @@ class DDLCommandSuite extends PlanTest {
comparePlans(parsed4, expected4)
}
+ test("create table - table file format") {
+ val allSources = Seq("parquet", "parquetfile", "orc", "orcfile", "avro", "avrofile",
+ "sequencefile", "rcfile", "textfile")
+
+ allSources.foreach { s =>
+ val query = s"CREATE TABLE my_tab STORED AS $s"
+ val ct = parseAs[CreateTableCommand](query)
+ val hiveSerde = HiveSerDe.sourceToSerDe(s, new SQLConf)
+ assert(hiveSerde.isDefined)
+ assert(ct.table.storage.serde == hiveSerde.get.serde)
+ assert(ct.table.storage.inputFormat == hiveSerde.get.inputFormat)
+ assert(ct.table.storage.outputFormat == hiveSerde.get.outputFormat)
+ }
+ }
+
test("create table - row format and table file format") {
val createTableStart = "CREATE TABLE my_tab ROW FORMAT"
val fileFormat = s"STORED AS INPUTFORMAT 'inputfmt' OUTPUTFORMAT 'outputfmt'"
--
GitLab