From eb10b481ca51a3e010c168a1fb0baf1a94dcadfa Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Fri, 3 Jun 2016 13:56:22 -0700
Subject: [PATCH] [SPARK-15286][SQL] Make the output readable for EXPLAIN
CREATE TABLE and DESC EXTENDED
#### What changes were proposed in this pull request?
Before this PR, the output of EXPLAIN of following SQL is like
```SQL
CREATE EXTERNAL TABLE extTable_with_partitions (key INT, value STRING)
PARTITIONED BY (ds STRING, hr STRING)
LOCATION '/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-b39a6185-8981-403b-a4aa-36fb2f4ca8a9'
```
``ExecutedCommand CreateTableCommand CatalogTable(`extTable_with_partitions`,CatalogTableType(EXTERNAL),CatalogStorageFormat(Some(/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-dd234718-e85d-4c5a-8353-8f1834ac0323),Some(org.apache.hadoop.mapred.TextInputFormat),Some(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat),None,false,Map()),List(CatalogColumn(key,int,true,None), CatalogColumn(value,string,true,None), CatalogColumn(ds,string,true,None), CatalogColumn(hr,string,true,None)),List(ds, hr),List(),List(),-1,,1463026413544,-1,Map(),None,None,None), false``
After this PR, the output is like
```
ExecutedCommand
: +- CreateTableCommand CatalogTable(
Table:`extTable_with_partitions`
Created:Thu Jun 02 21:30:54 PDT 2016
Last Access:Wed Dec 31 15:59:59 PST 1969
Type:EXTERNAL
Schema:[`key` int, `value` string, `ds` string, `hr` string]
Partition Columns:[`ds`, `hr`]
Storage(Location:/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-a06083b8-8e88-4d07-9ff0-d6bd8d943ad3, InputFormat:org.apache.hadoop.mapred.TextInputFormat, OutputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat)), false
```
This is also applicable to `DESC EXTENDED`. However, this does not have special handling for Data Source Tables. If needed, we need to move the logics of `DDLUtil`. Let me know if we should do it in this PR. Thanks! rxin liancheng
#### How was this patch tested?
Manual testing
Author: gatorsmile <gatorsmile@gmail.com>
Closes #13070 from gatorsmile/betterExplainCatalogTable.
---
.../sql/catalyst/catalog/interface.scala | 60 ++++++++++++++++++-
1 file changed, 58 insertions(+), 2 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index b59195770c..6197acab33 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.catalyst.catalog
+import java.util.Date
import javax.annotation.Nullable
import org.apache.spark.sql.AnalysisException
@@ -48,7 +49,26 @@ case class CatalogStorageFormat(
outputFormat: Option[String],
serde: Option[String],
compressed: Boolean,
- serdeProperties: Map[String, String])
+ serdeProperties: Map[String, String]) {
+
+ override def toString: String = {
+ val serdePropsToString =
+ if (serdeProperties.nonEmpty) {
+ s"Properties: " + serdeProperties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
+ } else {
+ ""
+ }
+ val output =
+ Seq(locationUri.map("Location: " + _).getOrElse(""),
+ inputFormat.map("InputFormat: " + _).getOrElse(""),
+ outputFormat.map("OutputFormat: " + _).getOrElse(""),
+ if (compressed) "Compressed" else "",
+ serde.map("Serde: " + _).getOrElse(""),
+ serdePropsToString)
+ output.filter(_.nonEmpty).mkString("Storage(", ", ", ")")
+ }
+
+}
object CatalogStorageFormat {
/** Empty storage format for default values and copies. */
@@ -65,8 +85,18 @@ case class CatalogColumn(
// as a string due to issues in converting Hive varchars to and from SparkSQL strings.
@Nullable dataType: String,
nullable: Boolean = true,
- comment: Option[String] = None)
+ comment: Option[String] = None) {
+
+ override def toString: String = {
+ val output =
+ Seq(s"`$name`",
+ dataType,
+ if (!nullable) "NOT NULL" else "",
+ comment.map("(" + _ + ")").getOrElse(""))
+ output.filter(_.nonEmpty).mkString(" ")
+ }
+}
/**
* A partition (Hive style) defined in the catalog.
@@ -140,6 +170,32 @@ case class CatalogTable(
locationUri, inputFormat, outputFormat, serde, compressed, serdeProperties))
}
+ override def toString: String = {
+ val tableProperties = properties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
+ val partitionColumns = partitionColumnNames.map("`" + _ + "`").mkString("[", ", ", "]")
+ val sortColumns = sortColumnNames.map("`" + _ + "`").mkString("[", ", ", "]")
+ val bucketColumns = bucketColumnNames.map("`" + _ + "`").mkString("[", ", ", "]")
+
+ val output =
+ Seq(s"Table: ${identifier.quotedString}",
+ if (owner.nonEmpty) s"Owner: $owner" else "",
+ s"Created: ${new Date(createTime).toString}",
+ s"Last Access: ${new Date(lastAccessTime).toString}",
+ s"Type: ${tableType.name}",
+ if (schema.nonEmpty) s"Schema: ${schema.mkString("[", ", ", "]")}" else "",
+ if (partitionColumnNames.nonEmpty) s"Partition Columns: $partitionColumns" else "",
+ if (numBuckets != -1) s"Num Buckets: $numBuckets" else "",
+ if (bucketColumnNames.nonEmpty) s"Bucket Columns: $bucketColumns" else "",
+ if (sortColumnNames.nonEmpty) s"Sort Columns: $sortColumns" else "",
+ viewOriginalText.map("Original View: " + _).getOrElse(""),
+ viewText.map("View: " + _).getOrElse(""),
+ comment.map("Comment: " + _).getOrElse(""),
+ if (properties.nonEmpty) s"Properties: $tableProperties" else "",
+ s"$storage")
+
+ output.filter(_.nonEmpty).mkString("CatalogTable(\n\t", "\n\t", ")")
+ }
+
}
--
GitLab