Skip to content
Snippets Groups Projects
Commit 372baf04 authored by Davies Liu's avatar Davies Liu Committed by Davies Liu
Browse files

[SPARK-14578] [SQL] Fix codegen for CreateExternalRow with nested wide schema

## What changes were proposed in this pull request?

The wide schema, the expression of fields will be splitted into multiple functions, but the variable for loopVar can't be accessed in splitted functions, this PR change them as class member.

## How was this patch tested?

Added regression test.

Author: Davies Liu <davies@databricks.com>

Closes #12338 from davies/nested_row.
parent d187e7de
No related branches found
No related tags found
No related merge requests found
......@@ -446,6 +446,8 @@ case class MapObjects private(
override def genCode(ctx: CodegenContext, ev: ExprCode): String = {
val javaType = ctx.javaType(dataType)
val elementJavaType = ctx.javaType(loopVar.dataType)
ctx.addMutableState("boolean", loopVar.isNull, "")
ctx.addMutableState(elementJavaType, loopVar.value, "")
val genInputData = inputData.gen(ctx)
val genFunction = lambdaFunction.gen(ctx)
val dataLength = ctx.freshName("dataLength")
......@@ -466,9 +468,9 @@ case class MapObjects private(
}
val loopNullCheck = if (primitiveElement) {
s"boolean ${loopVar.isNull} = ${genInputData.value}.isNullAt($loopIndex);"
s"${loopVar.isNull} = ${genInputData.value}.isNullAt($loopIndex);"
} else {
s"boolean ${loopVar.isNull} = ${genInputData.isNull} || ${loopVar.value} == null;"
s"${loopVar.isNull} = ${genInputData.isNull} || ${loopVar.value} == null;"
}
s"""
......@@ -484,7 +486,7 @@ case class MapObjects private(
int $loopIndex = 0;
while ($loopIndex < $dataLength) {
$elementJavaType ${loopVar.value} =
${loopVar.value} =
($elementJavaType)${genInputData.value}${itemAccessor(loopIndex)};
$loopNullCheck
......
......@@ -1664,4 +1664,19 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {
)
}
}
test("wide nested json table") {
val nested = (1 to 100).map { i =>
s"""
|"c$i": $i
""".stripMargin
}.mkString(", ")
val json = s"""
|{"a": [{$nested}], "b": [{$nested}]}
""".stripMargin
val rdd = sqlContext.sparkContext.makeRDD(Seq(json))
val df = sqlContext.read.json(rdd)
assert(df.schema.size === 2)
df.collect()
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment