Skip to content
Snippets Groups Projects
Commit eb10b481 authored by gatorsmile's avatar gatorsmile Committed by Wenchen Fan
Browse files

[SPARK-15286][SQL] Make the output readable for EXPLAIN CREATE TABLE and DESC EXTENDED

#### What changes were proposed in this pull request?
Before this PR, the output of EXPLAIN of following SQL is like

```SQL
CREATE EXTERNAL TABLE extTable_with_partitions (key INT, value STRING)
PARTITIONED BY (ds STRING, hr STRING)
LOCATION '/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-b39a6185-8981-403b-a4aa-36fb2f4ca8a9'
```
``ExecutedCommand CreateTableCommand CatalogTable(`extTable_with_partitions`,CatalogTableType(EXTERNAL),CatalogStorageFormat(Some(/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-dd234718-e85d-4c5a-8353-8f1834ac0323),Some(org.apache.hadoop.mapred.TextInputFormat),Some(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat),None,false,Map()),List(CatalogColumn(key,int,true,None), CatalogColumn(value,string,true,None), CatalogColumn(ds,string,true,None), CatalogColumn(hr,string,true,None)),List(ds, hr),List(),List(),-1,,1463026413544,-1,Map(),None,None,None), false``

After this PR, the output is like

```
ExecutedCommand
:  +- CreateTableCommand CatalogTable(
	Table:`extTable_with_partitions`
	Created:Thu Jun 02 21:30:54 PDT 2016
	Last Access:Wed Dec 31 15:59:59 PST 1969
	Type:EXTERNAL
	Schema:[`key` int, `value` string, `ds` string, `hr` string]
	Partition Columns:[`ds`, `hr`]
	Storage(Location:/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-a06083b8-8e88-4d07-9ff0-d6bd8d943ad3, InputFormat:org.apache.hadoop.mapred.TextInputFormat, OutputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat)), false
```

This is also applicable to `DESC EXTENDED`. However, this does not have special handling for Data Source Tables. If needed, we need to move the logics of `DDLUtil`. Let me know if we should do it in this PR. Thanks! rxin liancheng

#### How was this patch tested?
Manual testing

Author: gatorsmile <gatorsmile@gmail.com>

Closes #13070 from gatorsmile/betterExplainCatalogTable.
parent e5269139
No related branches found
No related tags found
No related merge requests found
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
package org.apache.spark.sql.catalyst.catalog package org.apache.spark.sql.catalyst.catalog
import java.util.Date
import javax.annotation.Nullable import javax.annotation.Nullable
import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.AnalysisException
...@@ -48,7 +49,26 @@ case class CatalogStorageFormat( ...@@ -48,7 +49,26 @@ case class CatalogStorageFormat(
outputFormat: Option[String], outputFormat: Option[String],
serde: Option[String], serde: Option[String],
compressed: Boolean, compressed: Boolean,
serdeProperties: Map[String, String]) serdeProperties: Map[String, String]) {
override def toString: String = {
val serdePropsToString =
if (serdeProperties.nonEmpty) {
s"Properties: " + serdeProperties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
} else {
""
}
val output =
Seq(locationUri.map("Location: " + _).getOrElse(""),
inputFormat.map("InputFormat: " + _).getOrElse(""),
outputFormat.map("OutputFormat: " + _).getOrElse(""),
if (compressed) "Compressed" else "",
serde.map("Serde: " + _).getOrElse(""),
serdePropsToString)
output.filter(_.nonEmpty).mkString("Storage(", ", ", ")")
}
}
object CatalogStorageFormat { object CatalogStorageFormat {
/** Empty storage format for default values and copies. */ /** Empty storage format for default values and copies. */
...@@ -65,8 +85,18 @@ case class CatalogColumn( ...@@ -65,8 +85,18 @@ case class CatalogColumn(
// as a string due to issues in converting Hive varchars to and from SparkSQL strings. // as a string due to issues in converting Hive varchars to and from SparkSQL strings.
@Nullable dataType: String, @Nullable dataType: String,
nullable: Boolean = true, nullable: Boolean = true,
comment: Option[String] = None) comment: Option[String] = None) {
override def toString: String = {
val output =
Seq(s"`$name`",
dataType,
if (!nullable) "NOT NULL" else "",
comment.map("(" + _ + ")").getOrElse(""))
output.filter(_.nonEmpty).mkString(" ")
}
}
/** /**
* A partition (Hive style) defined in the catalog. * A partition (Hive style) defined in the catalog.
...@@ -140,6 +170,32 @@ case class CatalogTable( ...@@ -140,6 +170,32 @@ case class CatalogTable(
locationUri, inputFormat, outputFormat, serde, compressed, serdeProperties)) locationUri, inputFormat, outputFormat, serde, compressed, serdeProperties))
} }
override def toString: String = {
val tableProperties = properties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
val partitionColumns = partitionColumnNames.map("`" + _ + "`").mkString("[", ", ", "]")
val sortColumns = sortColumnNames.map("`" + _ + "`").mkString("[", ", ", "]")
val bucketColumns = bucketColumnNames.map("`" + _ + "`").mkString("[", ", ", "]")
val output =
Seq(s"Table: ${identifier.quotedString}",
if (owner.nonEmpty) s"Owner: $owner" else "",
s"Created: ${new Date(createTime).toString}",
s"Last Access: ${new Date(lastAccessTime).toString}",
s"Type: ${tableType.name}",
if (schema.nonEmpty) s"Schema: ${schema.mkString("[", ", ", "]")}" else "",
if (partitionColumnNames.nonEmpty) s"Partition Columns: $partitionColumns" else "",
if (numBuckets != -1) s"Num Buckets: $numBuckets" else "",
if (bucketColumnNames.nonEmpty) s"Bucket Columns: $bucketColumns" else "",
if (sortColumnNames.nonEmpty) s"Sort Columns: $sortColumns" else "",
viewOriginalText.map("Original View: " + _).getOrElse(""),
viewText.map("View: " + _).getOrElse(""),
comment.map("Comment: " + _).getOrElse(""),
if (properties.nonEmpty) s"Properties: $tableProperties" else "",
s"$storage")
output.filter(_.nonEmpty).mkString("CatalogTable(\n\t", "\n\t", ")")
}
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment