From c78e2080e00a73159ab749691ad634fa6c0a2302 Mon Sep 17 00:00:00 2001
From: Jakob Odersky <jodersky@gmail.com>
Date: Tue, 19 Jan 2016 12:31:03 -0800
Subject: [PATCH] [SPARK-12816][SQL] De-alias type when generating schemas
Call `dealias` on local types to fix schema generation for abstract type members, such as
```scala
type KeyValue = (Int, String)
```
Add simple test
Author: Jakob Odersky <jodersky@gmail.com>
Closes #10749 from jodersky/aliased-schema.
---
.../org/apache/spark/sql/catalyst/ScalaReflection.scala | 5 ++++-
.../apache/spark/sql/catalyst/ScalaReflectionSuite.scala | 8 ++++++++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 79f723cf9b..643228d0eb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -642,7 +642,10 @@ trait ScalaReflection {
*
* @see SPARK-5281
*/
- def localTypeOf[T: TypeTag]: `Type` = typeTag[T].in(mirror).tpe
+ def localTypeOf[T: TypeTag]: `Type` = {
+ val tag = implicitly[TypeTag[T]]
+ tag.in(mirror).tpe.normalize
+ }
/** Returns a catalyst DataType and its nullability for the given Scala Type using reflection. */
def schemaFor(tpe: `Type`): Schema = ScalaReflectionLock.synchronized {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index c2aace1ef2..a32f5b70a0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -69,6 +69,10 @@ case class ComplexData(
case class GenericData[A](
genericField: A)
+object GenericData {
+ type IntData = GenericData[Int]
+}
+
case class MultipleConstructorsData(a: Int, b: String, c: Double) {
def this(b: String, a: Int) = this(a, b, c = 1.0)
}
@@ -186,6 +190,10 @@ class ScalaReflectionSuite extends SparkFunSuite {
nullable = true))
}
+ test("type-aliased data") {
+ assert(schemaFor[GenericData[Int]] == schemaFor[GenericData.IntData])
+ }
+
test("convert PrimitiveData to catalyst") {
val data = PrimitiveData(1, 1, 1, 1, 1, 1, true)
val convertedData = InternalRow(1, 1.toLong, 1.toDouble, 1.toFloat, 1.toShort, 1.toByte, true)
--
GitLab