From df766a471426625fe86c8845f6261e0fc087772d Mon Sep 17 00:00:00 2001
From: Xiao Li <gatorsmile@gmail.com>
Date: Wed, 14 Jun 2017 11:13:16 -0700
Subject: [PATCH] [SPARK-21089][SQL] Fix DESC EXTENDED/FORMATTED to Show Table
Properties
### What changes were proposed in this pull request?
Since both table properties and storage properties share the same key values, table properties are not shown in the output of DESC EXTENDED/FORMATTED when the storage properties are not empty.
This PR is to fix the above issue by renaming them to different keys.
### How was this patch tested?
Added test cases.
Author: Xiao Li <gatorsmile@gmail.com>
Closes #18294 from gatorsmile/tableProperties.
---
.../sql/catalyst/catalog/interface.scala | 4 +-
.../resources/sql-tests/inputs/describe.sql | 3 +
.../describe-table-after-alter-table.sql.out | 2 +-
.../sql-tests/results/describe.sql.out | 183 ++++++++++--------
4 files changed, 105 insertions(+), 87 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 2f328ccc49..c043ed9c43 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -75,7 +75,7 @@ case class CatalogStorageFormat(
CatalogUtils.maskCredentials(properties) match {
case props if props.isEmpty => // No-op
case props =>
- map.put("Properties", props.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]"))
+ map.put("Storage Properties", props.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]"))
}
map
}
@@ -316,7 +316,7 @@ case class CatalogTable(
}
}
- if (properties.nonEmpty) map.put("Properties", tableProperties)
+ if (properties.nonEmpty) map.put("Table Properties", tableProperties)
stats.foreach(s => map.put("Statistics", s.simpleString))
map ++= storage.toLinkedHashMap
if (tracksPartitionsInCatalog) map.put("Partition Provider", "Catalog")
diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe.sql b/sql/core/src/test/resources/sql-tests/inputs/describe.sql
index 6de4cf0d5a..91b966829f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/describe.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/describe.sql
@@ -1,4 +1,5 @@
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
+ OPTIONS (a '1', b '2')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment';
@@ -13,6 +14,8 @@ CREATE TEMPORARY VIEW temp_Data_Source_View
CREATE VIEW v AS SELECT * FROM t;
+ALTER TABLE t SET TBLPROPERTIES (e = '3');
+
ALTER TABLE t ADD PARTITION (c='Us', d=1);
DESCRIBE t;
diff --git a/sql/core/src/test/resources/sql-tests/results/describe-table-after-alter-table.sql.out b/sql/core/src/test/resources/sql-tests/results/describe-table-after-alter-table.sql.out
index eece00d603..4bf4633491 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe-table-after-alter-table.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe-table-after-alter-table.sql.out
@@ -57,7 +57,7 @@ Last Access [not included in comparison]
Type MANAGED
Provider parquet
Comment modified comment
-Properties [type=parquet]
+Table Properties [type=parquet]
Location [not included in comparison]sql/core/spark-warehouse/table_with_comment
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index 46d32bbc52..329532cd7c 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -1,9 +1,10 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 31
+-- Number of queries: 32
-- !query 0
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
+ OPTIONS (a '1', b '2')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment'
-- !query 0 schema
@@ -42,7 +43,7 @@ struct<>
-- !query 4
-ALTER TABLE t ADD PARTITION (c='Us', d=1)
+ALTER TABLE t SET TBLPROPERTIES (e = '3')
-- !query 4 schema
struct<>
-- !query 4 output
@@ -50,10 +51,18 @@ struct<>
-- !query 5
-DESCRIBE t
+ALTER TABLE t ADD PARTITION (c='Us', d=1)
-- !query 5 schema
-struct<col_name:string,data_type:string,comment:string>
+struct<>
-- !query 5 output
+
+
+
+-- !query 6
+DESCRIBE t
+-- !query 6 schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query 6 output
a string
b int
c string
@@ -64,11 +73,11 @@ c string
d string
--- !query 6
+-- !query 7
DESC default.t
--- !query 6 schema
+-- !query 7 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 6 output
+-- !query 7 output
a string
b int
c string
@@ -79,11 +88,11 @@ c string
d string
--- !query 7
+-- !query 8
DESC TABLE t
--- !query 7 schema
+-- !query 8 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 7 output
+-- !query 8 output
a string
b int
c string
@@ -94,11 +103,11 @@ c string
d string
--- !query 8
+-- !query 9
DESC FORMATTED t
--- !query 8 schema
+-- !query 9 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 8 output
+-- !query 9 output
a string
b int
c string
@@ -118,16 +127,17 @@ Provider parquet
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
-Comment table_comment
+Table Properties [e=3]
Location [not included in comparison]sql/core/spark-warehouse/t
+Storage Properties [a=1, b=2]
Partition Provider Catalog
--- !query 9
+-- !query 10
DESC EXTENDED t
--- !query 9 schema
+-- !query 10 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 9 output
+-- !query 10 output
a string
b int
c string
@@ -147,16 +157,17 @@ Provider parquet
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
-Comment table_comment
+Table Properties [e=3]
Location [not included in comparison]sql/core/spark-warehouse/t
+Storage Properties [a=1, b=2]
Partition Provider Catalog
--- !query 10
+-- !query 11
DESC t PARTITION (c='Us', d=1)
--- !query 10 schema
+-- !query 11 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 10 output
+-- !query 11 output
a string
b int
c string
@@ -167,11 +178,11 @@ c string
d string
--- !query 11
+-- !query 12
DESC EXTENDED t PARTITION (c='Us', d=1)
--- !query 11 schema
+-- !query 12 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 11 output
+-- !query 12 output
a string
b int
c string
@@ -186,19 +197,21 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1
+Storage Properties [a=1, b=2]
# Storage Information
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
-Location [not included in comparison]sql/core/spark-warehouse/t
+Location [not included in comparison]sql/core/spark-warehouse/t
+Storage Properties [a=1, b=2]
--- !query 12
+-- !query 13
DESC FORMATTED t PARTITION (c='Us', d=1)
--- !query 12 schema
+-- !query 13 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 12 output
+-- !query 13 output
a string
b int
c string
@@ -213,39 +226,41 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1
+Storage Properties [a=1, b=2]
# Storage Information
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
-Location [not included in comparison]sql/core/spark-warehouse/t
+Location [not included in comparison]sql/core/spark-warehouse/t
+Storage Properties [a=1, b=2]
--- !query 13
+-- !query 14
DESC t PARTITION (c='Us', d=2)
--- !query 13 schema
+-- !query 14 schema
struct<>
--- !query 13 output
+-- !query 14 output
org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException
Partition not found in table 't' database 'default':
c -> Us
d -> 2;
--- !query 14
+-- !query 15
DESC t PARTITION (c='Us')
--- !query 14 schema
+-- !query 15 schema
struct<>
--- !query 14 output
+-- !query 15 output
org.apache.spark.sql.AnalysisException
Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`';
--- !query 15
+-- !query 16
DESC t PARTITION (c='Us', d)
--- !query 15 schema
+-- !query 16 schema
struct<>
--- !query 15 output
+-- !query 16 output
org.apache.spark.sql.catalyst.parser.ParseException
PARTITION specification is incomplete: `d`(line 1, pos 0)
@@ -255,19 +270,8 @@ DESC t PARTITION (c='Us', d)
^^^
--- !query 16
-DESC temp_v
--- !query 16 schema
-struct<col_name:string,data_type:string,comment:string>
--- !query 16 output
-a string
-b int
-c string
-d string
-
-
-- !query 17
-DESC TABLE temp_v
+DESC temp_v
-- !query 17 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 17 output
@@ -278,7 +282,7 @@ d string
-- !query 18
-DESC FORMATTED temp_v
+DESC TABLE temp_v
-- !query 18 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 18 output
@@ -289,7 +293,7 @@ d string
-- !query 19
-DESC EXTENDED temp_v
+DESC FORMATTED temp_v
-- !query 19 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 19 output
@@ -300,10 +304,21 @@ d string
-- !query 20
-DESC temp_Data_Source_View
+DESC EXTENDED temp_v
-- !query 20 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 20 output
+a string
+b int
+c string
+d string
+
+
+-- !query 21
+DESC temp_Data_Source_View
+-- !query 21 schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query 21 output
intType int test comment test1
stringType string
dateType date
@@ -322,42 +337,42 @@ arrayType array<string>
structType struct<f1:string,f2:int>
--- !query 21
+-- !query 22
DESC temp_v PARTITION (c='Us', d=1)
--- !query 21 schema
+-- !query 22 schema
struct<>
--- !query 21 output
+-- !query 22 output
org.apache.spark.sql.AnalysisException
DESC PARTITION is not allowed on a temporary view: temp_v;
--- !query 22
+-- !query 23
DESC v
--- !query 22 schema
+-- !query 23 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 22 output
+-- !query 23 output
a string
b int
c string
d string
--- !query 23
+-- !query 24
DESC TABLE v
--- !query 23 schema
+-- !query 24 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 23 output
+-- !query 24 output
a string
b int
c string
d string
--- !query 24
+-- !query 25
DESC FORMATTED v
--- !query 24 schema
+-- !query 25 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 24 output
+-- !query 25 output
a string
b int
c string
@@ -372,14 +387,14 @@ Type VIEW
View Text SELECT * FROM t
View Default Database default
View Query Output Columns [a, b, c, d]
-Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
+Table Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
--- !query 25
+-- !query 26
DESC EXTENDED v
--- !query 25 schema
+-- !query 26 schema
struct<col_name:string,data_type:string,comment:string>
--- !query 25 output
+-- !query 26 output
a string
b int
c string
@@ -394,28 +409,20 @@ Type VIEW
View Text SELECT * FROM t
View Default Database default
View Query Output Columns [a, b, c, d]
-Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
-
-
--- !query 26
-DESC v PARTITION (c='Us', d=1)
--- !query 26 schema
-struct<>
--- !query 26 output
-org.apache.spark.sql.AnalysisException
-DESC PARTITION is not allowed on a view: v;
+Table Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
-- !query 27
-DROP TABLE t
+DESC v PARTITION (c='Us', d=1)
-- !query 27 schema
struct<>
-- !query 27 output
-
+org.apache.spark.sql.AnalysisException
+DESC PARTITION is not allowed on a view: v;
-- !query 28
-DROP VIEW temp_v
+DROP TABLE t
-- !query 28 schema
struct<>
-- !query 28 output
@@ -423,7 +430,7 @@ struct<>
-- !query 29
-DROP VIEW temp_Data_Source_View
+DROP VIEW temp_v
-- !query 29 schema
struct<>
-- !query 29 output
@@ -431,8 +438,16 @@ struct<>
-- !query 30
-DROP VIEW v
+DROP VIEW temp_Data_Source_View
-- !query 30 schema
struct<>
-- !query 30 output
+
+
+-- !query 31
+DROP VIEW v
+-- !query 31 schema
+struct<>
+-- !query 31 output
+
--
GitLab