From 0489cee6b24ca34f1adab03a75d157e04a9e06b7 Mon Sep 17 00:00:00 2001
From: Yin Huai <huai@cse.ohio-state.edu>
Date: Fri, 8 Aug 2014 11:10:11 -0700
Subject: [PATCH] [SPARK-2908] [SQL] JsonRDD.nullTypeToStringType does not
convert all NullType to StringType
JIRA: https://issues.apache.org/jira/browse/SPARK-2908
Author: Yin Huai <huai@cse.ohio-state.edu>
Closes #1840 from yhuai/SPARK-2908 and squashes the following commits:
86e833e [Yin Huai] Update test.
cb11759 [Yin Huai] nullTypeToStringType should check columns with the type of array of structs.
---
.../scala/org/apache/spark/sql/json/JsonRDD.scala | 4 +++-
.../scala/org/apache/spark/sql/json/JsonSuite.scala | 11 ++++++++---
.../org/apache/spark/sql/json/TestJsonData.scala | 2 +-
3 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
index a3d2a1c7a5..1c0b03c684 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
@@ -109,7 +109,9 @@ private[sql] object JsonRDD extends Logging {
val newType = dataType match {
case NullType => StringType
case ArrayType(NullType, containsNull) => ArrayType(StringType, containsNull)
- case struct: StructType => nullTypeToStringType(struct)
+ case ArrayType(struct: StructType, containsNull) =>
+ ArrayType(nullTypeToStringType(struct), containsNull)
+ case struct: StructType =>nullTypeToStringType(struct)
case other: DataType => other
}
StructField(fieldName, newType, nullable)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
index 75c0589eb2..58b1e23891 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
@@ -213,7 +213,8 @@ class JsonSuite extends QueryTest {
StructField("arrayOfStruct", ArrayType(
StructType(
StructField("field1", BooleanType, true) ::
- StructField("field2", StringType, true) :: Nil)), true) ::
+ StructField("field2", StringType, true) ::
+ StructField("field3", StringType, true) :: Nil)), true) ::
StructField("struct", StructType(
StructField("field1", BooleanType, true) ::
StructField("field2", DecimalType, true) :: Nil), true) ::
@@ -263,8 +264,12 @@ class JsonSuite extends QueryTest {
// Access elements of an array of structs.
checkAnswer(
- sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2] from jsonTable"),
- (true :: "str1" :: Nil, false :: null :: Nil, null) :: Nil
+ sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
+ "from jsonTable"),
+ (true :: "str1" :: null :: Nil,
+ false :: null :: null :: Nil,
+ null :: null :: null :: Nil,
+ null) :: Nil
)
// Access a struct and fields inside of it.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/TestJsonData.scala b/sql/core/src/test/scala/org/apache/spark/sql/json/TestJsonData.scala
index d0180f3754..a88310b5f1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/json/TestJsonData.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/json/TestJsonData.scala
@@ -43,7 +43,7 @@ object TestJsonData {
"arrayOfDouble":[1.2, 1.7976931348623157E308, 4.9E-324, 2.2250738585072014E-308],
"arrayOfBoolean":[true, false, true],
"arrayOfNull":[null, null, null, null],
- "arrayOfStruct":[{"field1": true, "field2": "str1"}, {"field1": false}],
+ "arrayOfStruct":[{"field1": true, "field2": "str1"}, {"field1": false}, {"field3": null}],
"arrayOfArray1":[[1, 2, 3], ["str1", "str2"]],
"arrayOfArray2":[[1, 2, 3], [1.1, 2.1, 3.1]]
}""" :: Nil)
--
GitLab