From b78e3849b20d0d09b7146efd7ce8f203ef67b890 Mon Sep 17 00:00:00 2001
From: Reynold Xin <rxin@databricks.com>
Date: Fri, 9 Jun 2017 18:29:33 -0700
Subject: [PATCH] [SPARK-21042][SQL] Document Dataset.union is resolution by
position
## What changes were proposed in this pull request?
Document Dataset.union is resolution by position, not by name, since this has been a confusing point for a lot of users.
## How was this patch tested?
N/A - doc only change.
Author: Reynold Xin <rxin@databricks.com>
Closes #18256 from rxin/SPARK-21042.
---
R/pkg/R/DataFrame.R | 1 +
python/pyspark/sql/dataframe.py | 13 +++++++++----
.../main/scala/org/apache/spark/sql/Dataset.scala | 14 ++++++++------
3 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 166b39813c..3b9d42d6e7 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -2646,6 +2646,7 @@ generateAliasesForIntersectedCols <- function (x, intersectedColNames, suffix) {
#' Input SparkDataFrames can have different schemas (names and data types).
#'
#' Note: This does not remove duplicate rows across the two SparkDataFrames.
+#' Also as standard in SQL, this function resolves columns by position (not by name).
#'
#' @param x A SparkDataFrame
#' @param y A SparkDataFrame
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 99abfcc556..8541403dfe 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -1175,18 +1175,23 @@ class DataFrame(object):
@since(2.0)
def union(self, other):
- """ Return a new :class:`DataFrame` containing union of rows in this
- frame and another frame.
+ """ Return a new :class:`DataFrame` containing union of rows in this and another frame.
This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union
(that does deduplication of elements), use this function followed by a distinct.
+
+ Also as standard in SQL, this function resolves columns by position (not by name).
"""
return DataFrame(self._jdf.union(other._jdf), self.sql_ctx)
@since(1.3)
def unionAll(self, other):
- """ Return a new :class:`DataFrame` containing union of rows in this
- frame and another frame.
+ """ Return a new :class:`DataFrame` containing union of rows in this and another frame.
+
+ This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union
+ (that does deduplication of elements), use this function followed by a distinct.
+
+ Also as standard in SQL, this function resolves columns by position (not by name).
.. note:: Deprecated in 2.0, use union instead.
"""
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index f7637e005f..d28ff7888d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -1734,10 +1734,11 @@ class Dataset[T] private[sql](
/**
* Returns a new Dataset containing union of rows in this Dataset and another Dataset.
- * This is equivalent to `UNION ALL` in SQL.
*
- * To do a SQL-style set union (that does deduplication of elements), use this function followed
- * by a [[distinct]].
+ * This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union (that does
+ * deduplication of elements), use this function followed by a [[distinct]].
+ *
+ * Also as standard in SQL, this function resolves columns by position (not by name).
*
* @group typedrel
* @since 2.0.0
@@ -1747,10 +1748,11 @@ class Dataset[T] private[sql](
/**
* Returns a new Dataset containing union of rows in this Dataset and another Dataset.
- * This is equivalent to `UNION ALL` in SQL.
*
- * To do a SQL-style set union (that does deduplication of elements), use this function followed
- * by a [[distinct]].
+ * This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union (that does
+ * deduplication of elements), use this function followed by a [[distinct]].
+ *
+ * Also as standard in SQL, this function resolves columns by position (not by name).
*
* @group typedrel
* @since 2.0.0
--
GitLab