diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index cd6f03a13d7c7f315509db5e346ed5a6dc19dc78..7e57ba6287bb860a2b53f64cebe285835c8c4216 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -194,6 +194,7 @@ setMethod("isLocal",
#' 20 characters will be truncated. However, if set greater than zero,
#' truncates strings longer than \code{truncate} characters and all cells
#' will be aligned right.
+#' @param vertical whether print output rows vertically (one line per column value).
#' @param ... further arguments to be passed to or from other methods.
#' @family SparkDataFrame functions
#' @aliases showDF,SparkDataFrame-method
@@ -210,12 +211,13 @@ setMethod("isLocal",
#' @note showDF since 1.4.0
setMethod("showDF",
signature(x = "SparkDataFrame"),
- function(x, numRows = 20, truncate = TRUE) {
+ function(x, numRows = 20, truncate = TRUE, vertical = FALSE) {
if (is.logical(truncate) && truncate) {
- s <- callJMethod(x@sdf, "showString", numToInt(numRows), numToInt(20))
+ s <- callJMethod(x@sdf, "showString", numToInt(numRows), numToInt(20), vertical)
} else {
truncate2 <- as.numeric(truncate)
- s <- callJMethod(x@sdf, "showString", numToInt(numRows), numToInt(truncate2))
+ s <- callJMethod(x@sdf, "showString", numToInt(numRows), numToInt(truncate2),
+ vertical)
}
cat(s)
})
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 774caf53f3a4b9ff3130a667aebec9005b2c165d..ff21bb5d2fb3fc5eebeace83e7c56e87579dd322 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -290,13 +290,15 @@ class DataFrame(object):
return self._jdf.isStreaming()
@since(1.3)
- def show(self, n=20, truncate=True):
+ def show(self, n=20, truncate=True, vertical=False):
"""Prints the first ``n`` rows to the console.
:param n: Number of rows to show.
:param truncate: If set to True, truncate strings longer than 20 chars by default.
If set to a number greater than one, truncates long strings to length ``truncate``
and align cells right.
+ :param vertical: If set to True, print output rows vertically (one line
+ per column value).
>>> df
DataFrame[age: int, name: string]
@@ -314,11 +316,18 @@ class DataFrame(object):
| 2| Ali|
| 5| Bob|
+---+----+
+ >>> df.show(vertical=True)
+ -RECORD 0-----
+ age | 2
+ name | Alice
+ -RECORD 1-----
+ age | 5
+ name | Bob
"""
if isinstance(truncate, bool) and truncate:
- print(self._jdf.showString(n, 20))
+ print(self._jdf.showString(n, 20, vertical))
else:
- print(self._jdf.showString(n, int(truncate)))
+ print(self._jdf.showString(n, int(truncate), vertical))
def __repr__(self):
return "DataFrame[%s]" % (", ".join("%s: %s" % c for c in self.dtypes))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 06dd5500718de0a9f063683547bbdaab21908e24..147e7651ce55ba9adf73b2a01dbc18a86df63979 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -240,8 +240,10 @@ class Dataset[T] private[sql](
* @param _numRows Number of rows to show
* @param truncate If set to more than 0, truncates strings to `truncate` characters and
* all cells will be aligned right.
+ * @param vertical If set to true, prints output rows vertically (one line per column value).
*/
- private[sql] def showString(_numRows: Int, truncate: Int = 20): String = {
+ private[sql] def showString(
+ _numRows: Int, truncate: Int = 20, vertical: Boolean = false): String = {
val numRows = _numRows.max(0)
val takeResult = toDF().take(numRows + 1)
val hasMoreData = takeResult.length > numRows
@@ -277,46 +279,80 @@ class Dataset[T] private[sql](
val sb = new StringBuilder
val numCols = schema.fieldNames.length
+ // We set a minimum column width at '3'
+ val minimumColWidth = 3
- // Initialise the width of each column to a minimum value of '3'
- val colWidths = Array.fill(numCols)(3)
+ if (!vertical) {
+ // Initialise the width of each column to a minimum value
+ val colWidths = Array.fill(numCols)(minimumColWidth)
- // Compute the width of each column
- for (row <- rows) {
- for ((cell, i) <- row.zipWithIndex) {
- colWidths(i) = math.max(colWidths(i), cell.length)
- }
- }
-
- // Create SeparateLine
- val sep: String = colWidths.map("-" * _).addString(sb, "+", "+", "+\n").toString()
-
- // column names
- rows.head.zipWithIndex.map { case (cell, i) =>
- if (truncate > 0) {
- StringUtils.leftPad(cell, colWidths(i))
- } else {
- StringUtils.rightPad(cell, colWidths(i))
+ // Compute the width of each column
+ for (row <- rows) {
+ for ((cell, i) <- row.zipWithIndex) {
+ colWidths(i) = math.max(colWidths(i), cell.length)
+ }
}
- }.addString(sb, "|", "|", "|\n")
- sb.append(sep)
+ // Create SeparateLine
+ val sep: String = colWidths.map("-" * _).addString(sb, "+", "+", "+\n").toString()
- // data
- rows.tail.map {
- _.zipWithIndex.map { case (cell, i) =>
+ // column names
+ rows.head.zipWithIndex.map { case (cell, i) =>
if (truncate > 0) {
- StringUtils.leftPad(cell.toString, colWidths(i))
+ StringUtils.leftPad(cell, colWidths(i))
} else {
- StringUtils.rightPad(cell.toString, colWidths(i))
+ StringUtils.rightPad(cell, colWidths(i))
}
}.addString(sb, "|", "|", "|\n")
- }
- sb.append(sep)
+ sb.append(sep)
+
+ // data
+ rows.tail.foreach {
+ _.zipWithIndex.map { case (cell, i) =>
+ if (truncate > 0) {
+ StringUtils.leftPad(cell.toString, colWidths(i))
+ } else {
+ StringUtils.rightPad(cell.toString, colWidths(i))
+ }
+ }.addString(sb, "|", "|", "|\n")
+ }
+
+ sb.append(sep)
+ } else {
+ // Extended display mode enabled
+ val fieldNames = rows.head
+ val dataRows = rows.tail
+
+ // Compute the width of field name and data columns
+ val fieldNameColWidth = fieldNames.foldLeft(minimumColWidth) { case (curMax, fieldName) =>
+ math.max(curMax, fieldName.length)
+ }
+ val dataColWidth = dataRows.foldLeft(minimumColWidth) { case (curMax, row) =>
+ math.max(curMax, row.map(_.length).reduceLeftOption[Int] { case (cellMax, cell) =>
+ math.max(cellMax, cell)
+ }.getOrElse(0))
+ }
+
+ dataRows.zipWithIndex.foreach { case (row, i) =>
+ // "+ 5" in size means a character length except for padded names and data
+ val rowHeader = StringUtils.rightPad(
+ s"-RECORD $i", fieldNameColWidth + dataColWidth + 5, "-")
+ sb.append(rowHeader).append("\n")
+ row.zipWithIndex.map { case (cell, j) =>
+ val fieldName = StringUtils.rightPad(fieldNames(j), fieldNameColWidth)
+ val data = StringUtils.rightPad(cell, dataColWidth)
+ s" $fieldName | $data "
+ }.addString(sb, "", "\n", "\n")
+ }
+ }
- // For Data that has more than "numRows" records
- if (hasMoreData) {
+ // Print a footer
+ if (vertical && data.isEmpty) {
+ // In a vertical mode, print an empty row set explicitly
+ sb.append("(0 rows)\n")
+ } else if (hasMoreData) {
+ // For Data that has more than "numRows" records
val rowsString = if (numRows == 1) "row" else "rows"
sb.append(s"only showing top $numRows $rowsString\n")
}
@@ -663,8 +699,59 @@ class Dataset[T] private[sql](
* @group action
* @since 1.6.0
*/
+ def show(numRows: Int, truncate: Int): Unit = show(numRows, truncate, vertical = false)
+
+ /**
+ * Displays the Dataset in a tabular form. For example:
+ * {{{
+ * year month AVG('Adj Close) MAX('Adj Close)
+ * 1980 12 0.503218 0.595103
+ * 1981 01 0.523289 0.570307
+ * 1982 02 0.436504 0.475256
+ * 1983 03 0.410516 0.442194
+ * 1984 04 0.450090 0.483521
+ * }}}
+ *
+ * If `vertical` enabled, this command prints output rows vertically (one line per column value)?
+ *
+ * {{{
+ * -RECORD 0-------------------
+ * year | 1980
+ * month | 12
+ * AVG('Adj Close) | 0.503218
+ * AVG('Adj Close) | 0.595103
+ * -RECORD 1-------------------
+ * year | 1981
+ * month | 01
+ * AVG('Adj Close) | 0.523289
+ * AVG('Adj Close) | 0.570307
+ * -RECORD 2-------------------
+ * year | 1982
+ * month | 02
+ * AVG('Adj Close) | 0.436504
+ * AVG('Adj Close) | 0.475256
+ * -RECORD 3-------------------
+ * year | 1983
+ * month | 03
+ * AVG('Adj Close) | 0.410516
+ * AVG('Adj Close) | 0.442194
+ * -RECORD 4-------------------
+ * year | 1984
+ * month | 04
+ * AVG('Adj Close) | 0.450090
+ * AVG('Adj Close) | 0.483521
+ * }}}
+ *
+ * @param numRows Number of rows to show
+ * @param truncate If set to more than 0, truncates strings to `truncate` characters and
+ * all cells will be aligned right.
+ * @param vertical If set to true, prints output rows vertically (one line per column value).
+ * @group action
+ * @since 2.3.0
+ */
// scalastyle:off println
- def show(numRows: Int, truncate: Int): Unit = println(showString(numRows, truncate))
+ def show(numRows: Int, truncate: Int, vertical: Boolean): Unit =
+ println(showString(numRows, truncate, vertical))
// scalastyle:on println
/**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index b4893b56a8a84c638432b42487d5b9a78ed3a7e2..ef0de6f6f4ff1dd74a8f195077e301e7728817fa 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -764,6 +764,21 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(df.showString(10, truncate = 20) === expectedAnswerForTrue)
}
+ test("showString: truncate = [0, 20], vertical = true") {
+ val longString = Array.fill(21)("1").mkString
+ val df = sparkContext.parallelize(Seq("1", longString)).toDF()
+ val expectedAnswerForFalse = "-RECORD 0----------------------\n" +
+ " value | 1 \n" +
+ "-RECORD 1----------------------\n" +
+ " value | 111111111111111111111 \n"
+ assert(df.showString(10, truncate = 0, vertical = true) === expectedAnswerForFalse)
+ val expectedAnswerForTrue = "-RECORD 0---------------------\n" +
+ " value | 1 \n" +
+ "-RECORD 1---------------------\n" +
+ " value | 11111111111111111... \n"
+ assert(df.showString(10, truncate = 20, vertical = true) === expectedAnswerForTrue)
+ }
+
test("showString: truncate = [3, 17]") {
val longString = Array.fill(21)("1").mkString
val df = sparkContext.parallelize(Seq("1", longString)).toDF()
@@ -785,6 +800,21 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(df.showString(10, truncate = 17) === expectedAnswerForTrue)
}
+ test("showString: truncate = [3, 17], vertical = true") {
+ val longString = Array.fill(21)("1").mkString
+ val df = sparkContext.parallelize(Seq("1", longString)).toDF()
+ val expectedAnswerForFalse = "-RECORD 0----\n" +
+ " value | 1 \n" +
+ "-RECORD 1----\n" +
+ " value | 111 \n"
+ assert(df.showString(10, truncate = 3, vertical = true) === expectedAnswerForFalse)
+ val expectedAnswerForTrue = "-RECORD 0------------------\n" +
+ " value | 1 \n" +
+ "-RECORD 1------------------\n" +
+ " value | 11111111111111... \n"
+ assert(df.showString(10, truncate = 17, vertical = true) === expectedAnswerForTrue)
+ }
+
test("showString(negative)") {
val expectedAnswer = """+---+-----+
||key|value|
@@ -795,6 +825,11 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(testData.select($"*").showString(-1) === expectedAnswer)
}
+ test("showString(negative), vertical = true") {
+ val expectedAnswer = "(0 rows)\n"
+ assert(testData.select($"*").showString(-1, vertical = true) === expectedAnswer)
+ }
+
test("showString(0)") {
val expectedAnswer = """+---+-----+
||key|value|
@@ -805,6 +840,11 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(testData.select($"*").showString(0) === expectedAnswer)
}
+ test("showString(0), vertical = true") {
+ val expectedAnswer = "(0 rows)\n"
+ assert(testData.select($"*").showString(0, vertical = true) === expectedAnswer)
+ }
+
test("showString: array") {
val df = Seq(
(Array(1, 2, 3), Array(1, 2, 3)),
@@ -820,6 +860,20 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(df.showString(10) === expectedAnswer)
}
+ test("showString: array, vertical = true") {
+ val df = Seq(
+ (Array(1, 2, 3), Array(1, 2, 3)),
+ (Array(2, 3, 4), Array(2, 3, 4))
+ ).toDF()
+ val expectedAnswer = "-RECORD 0--------\n" +
+ " _1 | [1, 2, 3] \n" +
+ " _2 | [1, 2, 3] \n" +
+ "-RECORD 1--------\n" +
+ " _1 | [2, 3, 4] \n" +
+ " _2 | [2, 3, 4] \n"
+ assert(df.showString(10, vertical = true) === expectedAnswer)
+ }
+
test("showString: binary") {
val df = Seq(
("12".getBytes(StandardCharsets.UTF_8), "ABC.".getBytes(StandardCharsets.UTF_8)),
@@ -835,6 +889,20 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(df.showString(10) === expectedAnswer)
}
+ test("showString: binary, vertical = true") {
+ val df = Seq(
+ ("12".getBytes(StandardCharsets.UTF_8), "ABC.".getBytes(StandardCharsets.UTF_8)),
+ ("34".getBytes(StandardCharsets.UTF_8), "12346".getBytes(StandardCharsets.UTF_8))
+ ).toDF()
+ val expectedAnswer = "-RECORD 0---------------\n" +
+ " _1 | [31 32] \n" +
+ " _2 | [41 42 43 2E] \n" +
+ "-RECORD 1---------------\n" +
+ " _1 | [33 34] \n" +
+ " _2 | [31 32 33 34 36] \n"
+ assert(df.showString(10, vertical = true) === expectedAnswer)
+ }
+
test("showString: minimum column width") {
val df = Seq(
(1, 1),
@@ -850,6 +918,20 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(df.showString(10) === expectedAnswer)
}
+ test("showString: minimum column width, vertical = true") {
+ val df = Seq(
+ (1, 1),
+ (2, 2)
+ ).toDF()
+ val expectedAnswer = "-RECORD 0--\n" +
+ " _1 | 1 \n" +
+ " _2 | 1 \n" +
+ "-RECORD 1--\n" +
+ " _1 | 2 \n" +
+ " _2 | 2 \n"
+ assert(df.showString(10, vertical = true) === expectedAnswer)
+ }
+
test("SPARK-7319 showString") {
val expectedAnswer = """+---+-----+
||key|value|
@@ -861,6 +943,14 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(testData.select($"*").showString(1) === expectedAnswer)
}
+ test("SPARK-7319 showString, vertical = true") {
+ val expectedAnswer = "-RECORD 0----\n" +
+ " key | 1 \n" +
+ " value | 1 \n" +
+ "only showing top 1 row\n"
+ assert(testData.select($"*").showString(1, vertical = true) === expectedAnswer)
+ }
+
test("SPARK-7327 show with empty dataFrame") {
val expectedAnswer = """+---+-----+
||key|value|
@@ -870,6 +960,10 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
assert(testData.select($"*").filter($"key" < 0).showString(1) === expectedAnswer)
}
+ test("SPARK-7327 show with empty dataFrame, vertical = true") {
+ assert(testData.select($"*").filter($"key" < 0).showString(1, vertical = true) === "(0 rows)\n")
+ }
+
test("SPARK-18350 show with session local timezone") {
val d = Date.valueOf("2016-12-01")
val ts = Timestamp.valueOf("2016-12-01 00:00:00")
@@ -894,6 +988,24 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
}
}
+ test("SPARK-18350 show with session local timezone, vertical = true") {
+ val d = Date.valueOf("2016-12-01")
+ val ts = Timestamp.valueOf("2016-12-01 00:00:00")
+ val df = Seq((d, ts)).toDF("d", "ts")
+ val expectedAnswer = "-RECORD 0------------------\n" +
+ " d | 2016-12-01 \n" +
+ " ts | 2016-12-01 00:00:00 \n"
+ assert(df.showString(1, truncate = 0, vertical = true) === expectedAnswer)
+
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+
+ val expectedAnswer = "-RECORD 0------------------\n" +
+ " d | 2016-12-01 \n" +
+ " ts | 2016-12-01 08:00:00 \n"
+ assert(df.showString(1, truncate = 0, vertical = true) === expectedAnswer)
+ }
+ }
+
test("createDataFrame(RDD[Row], StructType) should convert UDTs (SPARK-6672)") {
val rowRDD = sparkContext.parallelize(Seq(Row(new ExamplePoint(1.0, 2.0))))
val schema = StructType(Array(StructField("point", new ExamplePointUDT(), false)))