Skip to content
Snippets Groups Projects
Commit dc1dbf20 authored by Cheng Lian's avatar Cheng Lian Committed by Michael Armbrust
Browse files

[SPARK-3414][SQL] Stores analyzed logical plan when registering a temp table

Case insensitivity breaks when unresolved relation contains attributes with uppercase letters in their names, because we store unanalyzed logical plan when registering temp tables while the `CaseInsensitivityAttributeReferences` batch runs before the `Resolution` batch. To fix this issue, we need to store analyzed logical plan.

Author: Cheng Lian <lian.cs.zju@gmail.com>

Closes #2293 from liancheng/spark-3414 and squashes the following commits:

d9fa1d6 [Cheng Lian] Stores analyzed logical plan when registering a temp table
parent ca0348e6
No related branches found
No related tags found
No related merge requests found
...@@ -246,7 +246,7 @@ class SQLContext(@transient val sparkContext: SparkContext) ...@@ -246,7 +246,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
* @group userf * @group userf
*/ */
def registerRDDAsTable(rdd: SchemaRDD, tableName: String): Unit = { def registerRDDAsTable(rdd: SchemaRDD, tableName: String): Unit = {
catalog.registerTable(None, tableName, rdd.logicalPlan) catalog.registerTable(None, tableName, rdd.queryExecution.analyzed)
} }
/** /**
...@@ -411,7 +411,7 @@ class SQLContext(@transient val sparkContext: SparkContext) ...@@ -411,7 +411,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
protected def stringOrError[A](f: => A): String = protected def stringOrError[A](f: => A): String =
try f.toString catch { case e: Throwable => e.toString } try f.toString catch { case e: Throwable => e.toString }
def simpleString: String = def simpleString: String =
s"""== Physical Plan == s"""== Physical Plan ==
|${stringOrError(executedPlan)} |${stringOrError(executedPlan)}
""" """
......
...@@ -17,11 +17,8 @@ ...@@ -17,11 +17,8 @@
package org.apache.spark.sql.hive.execution package org.apache.spark.sql.hive.execution
import java.io.File
import scala.util.Try import scala.util.Try
import org.apache.spark.SparkException
import org.apache.spark.sql.hive._ import org.apache.spark.sql.hive._
import org.apache.spark.sql.hive.test.TestHive import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.hive.test.TestHive._ import org.apache.spark.sql.hive.test.TestHive._
...@@ -514,6 +511,28 @@ class HiveQuerySuite extends HiveComparisonTest { ...@@ -514,6 +511,28 @@ class HiveQuerySuite extends HiveComparisonTest {
sql("DROP TABLE alter1") sql("DROP TABLE alter1")
} }
case class LogEntry(filename: String, message: String)
case class LogFile(name: String)
test("SPARK-3414 regression: should store analyzed logical plan when registering a temp table") {
sparkContext.makeRDD(Seq.empty[LogEntry]).registerTempTable("rawLogs")
sparkContext.makeRDD(Seq.empty[LogFile]).registerTempTable("logFiles")
sql(
"""
SELECT name, message
FROM rawLogs
JOIN (
SELECT name
FROM logFiles
) files
ON rawLogs.filename = files.name
""").registerTempTable("boom")
// This should be successfully analyzed
sql("SELECT * FROM boom").queryExecution.analyzed
}
test("parse HQL set commands") { test("parse HQL set commands") {
// Adapted from its SQL counterpart. // Adapted from its SQL counterpart.
val testKey = "spark.sql.key.usedfortestonly" val testKey = "spark.sql.key.usedfortestonly"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment