From f39621c998a0fe91a5115f3f843c3ca8dd71c1ab Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sat, 21 May 2016 08:11:14 -0700
Subject: [PATCH] [SPARK-15462][SQL][TEST] unresolved === false` is enough in
testcases.
## What changes were proposed in this pull request?
In only `catalyst` module, there exists 8 evaluation test cases on unresolved expressions. But, in real-world situation, those cases doesn't happen since they occurs exceptions before evaluations.
```scala
scala> sql("select format_number(null, 3)")
res0: org.apache.spark.sql.DataFrame = [format_number(CAST(NULL AS DOUBLE), 3): string]
scala> sql("select format_number(cast(null as NULL), 3)")
org.apache.spark.sql.catalyst.parser.ParseException:
DataType null() is not supported.(line 1, pos 34)
```
This PR makes those testcases more realistic.
```scala
- checkEvaluation(FormatNumber(Literal.create(null, NullType), Literal(3)), null)
+ assert(FormatNumber(Literal.create(null, NullType), Literal(3)).resolved === false)
```
Also, this PR also removes redundant `resolved` checking in `FoldablePropagation` optimizer.
## How was this patch tested?
Pass the modified Jenkins tests.
Author: Dongjoon Hyun <dongjoon@apache.org>
Closes #13241 from dongjoon-hyun/SPARK-15462.
---
.../apache/spark/sql/catalyst/optimizer/Optimizer.scala | 2 +-
.../apache/spark/sql/catalyst/expressions/CastSuite.scala | 7 -------
.../sql/catalyst/expressions/DateExpressionsSuite.scala | 6 +++---
.../sql/catalyst/expressions/StringExpressionsSuite.scala | 2 +-
4 files changed, 5 insertions(+), 12 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index a6fb34cbfb..5e998d6188 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -673,7 +673,7 @@ object FoldablePropagation extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = {
val foldableMap = AttributeMap(plan.flatMap {
case Project(projectList, _) => projectList.collect {
- case a: Alias if a.resolved && a.child.foldable => (a.toAttribute, a)
+ case a: Alias if a.child.foldable => (a.toAttribute, a)
}
case _ => Nil
})
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index 43af359207..dfda7c50f2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -548,7 +548,6 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
{
val ret = cast(array_notNull, ArrayType(BooleanType, containsNull = false))
assert(ret.resolved === false)
- checkEvaluation(ret, Seq(null, true, false))
}
{
@@ -607,7 +606,6 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
{
val ret = cast(map_notNull, MapType(StringType, BooleanType, valueContainsNull = false))
assert(ret.resolved === false)
- checkEvaluation(ret, Map("a" -> null, "b" -> true, "c" -> false))
}
{
val ret = cast(map_notNull, MapType(IntegerType, StringType, valueContainsNull = true))
@@ -714,7 +712,6 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
StructField("b", BooleanType, nullable = true),
StructField("c", BooleanType, nullable = false))))
assert(ret.resolved === false)
- checkEvaluation(ret, InternalRow(null, true, false))
}
{
@@ -755,10 +752,6 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
StructField("l", LongType, nullable = true)))))))
assert(ret.resolved === false)
- checkEvaluation(ret, Row(
- Seq(123, null, null),
- Map("a" -> null, "b" -> true, "c" -> false),
- Row(0L)))
}
test("cast between string and interval") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 53c66d8a75..6118a34d29 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -143,7 +143,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}
test("Seconds") {
- checkEvaluation(Second(Literal.create(null, DateType)), null)
+ assert(Second(Literal.create(null, DateType)).resolved === false)
checkEvaluation(Second(Cast(Literal(d), TimestampType)), 0)
checkEvaluation(Second(Cast(Literal(sdf.format(d)), TimestampType)), 15)
checkEvaluation(Second(Literal(ts)), 15)
@@ -176,7 +176,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}
test("Hour") {
- checkEvaluation(Hour(Literal.create(null, DateType)), null)
+ assert(Hour(Literal.create(null, DateType)).resolved === false)
checkEvaluation(Hour(Cast(Literal(d), TimestampType)), 0)
checkEvaluation(Hour(Cast(Literal(sdf.format(d)), TimestampType)), 13)
checkEvaluation(Hour(Literal(ts)), 13)
@@ -195,7 +195,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}
test("Minute") {
- checkEvaluation(Minute(Literal.create(null, DateType)), null)
+ assert(Minute(Literal.create(null, DateType)).resolved === false)
checkEvaluation(Minute(Cast(Literal(d), TimestampType)), 0)
checkEvaluation(Minute(Cast(Literal(sdf.format(d)), TimestampType)), 10)
checkEvaluation(Minute(Literal(ts)), 10)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index 2cf8ca7000..c09c64fd6b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -688,7 +688,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
Literal(Decimal(123123324123L) * Decimal(123123.21234d)), Literal(4)),
"15,159,339,180,002,773.2778")
checkEvaluation(FormatNumber(Literal.create(null, IntegerType), Literal(3)), null)
- checkEvaluation(FormatNumber(Literal.create(null, NullType), Literal(3)), null)
+ assert(FormatNumber(Literal.create(null, NullType), Literal(3)).resolved === false)
}
test("find in set") {
--
GitLab