[SPARK-9432][SQL] Audit expression unit tests to make sure we pass the proper numeric ranges

JIRA: https://issues.apache.org/jira/browse/SPARK-9432

Author: Yijie Shen <henry.yijieshen@gmail.com>

Closes #7933 from yjshen/numeric_ranges and squashes the following commits:

e719f78 [Yijie Shen] proper integral range check

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala

@@ -23,6 +23,8 @@ import org.apache.spark.sql.types._
 
 class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
 
+  import IntegralLiteralTestUtils._
+
   /**
    * Runs through the testFunc for all numeric data types.
    *
@@ -47,6 +49,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
       checkEvaluation(Add(Literal.create(null, left.dataType), right), null)
       checkEvaluation(Add(left, Literal.create(null, right.dataType)), null)
     }
+    checkEvaluation(Add(positiveShortLit, negativeShortLit), -1.toShort)
+    checkEvaluation(Add(positiveIntLit, negativeIntLit), -1)
+    checkEvaluation(Add(positiveLongLit, negativeLongLit), -1L)
   }
 
   test("- (UnaryMinus)") {
@@ -60,6 +65,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(UnaryMinus(Literal(Int.MinValue)), Int.MinValue)
     checkEvaluation(UnaryMinus(Literal(Short.MinValue)), Short.MinValue)
     checkEvaluation(UnaryMinus(Literal(Byte.MinValue)), Byte.MinValue)
+    checkEvaluation(UnaryMinus(positiveShortLit), (- positiveShort).toShort)
+    checkEvaluation(UnaryMinus(negativeShortLit), (- negativeShort).toShort)
+    checkEvaluation(UnaryMinus(positiveIntLit), - positiveInt)
+    checkEvaluation(UnaryMinus(negativeIntLit), - negativeInt)
+    checkEvaluation(UnaryMinus(positiveLongLit), - positiveLong)
+    checkEvaluation(UnaryMinus(negativeLongLit), - negativeLong)
   }
 
   test("- (Minus)") {
@@ -70,6 +81,10 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
       checkEvaluation(Subtract(Literal.create(null, left.dataType), right), null)
       checkEvaluation(Subtract(left, Literal.create(null, right.dataType)), null)
     }
+    checkEvaluation(Subtract(positiveShortLit, negativeShortLit),
+      (positiveShort - negativeShort).toShort)
+    checkEvaluation(Subtract(positiveIntLit, negativeIntLit), positiveInt - negativeInt)
+    checkEvaluation(Subtract(positiveLongLit, negativeLongLit), positiveLong - negativeLong)
   }
 
   test("* (Multiply)") {
@@ -80,6 +95,10 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
       checkEvaluation(Multiply(Literal.create(null, left.dataType), right), null)
       checkEvaluation(Multiply(left, Literal.create(null, right.dataType)), null)
     }
+    checkEvaluation(Multiply(positiveShortLit, negativeShortLit),
+      (positiveShort * negativeShort).toShort)
+    checkEvaluation(Multiply(positiveIntLit, negativeIntLit), positiveInt * negativeInt)
+    checkEvaluation(Multiply(positiveLongLit, negativeLongLit), positiveLong * negativeLong)
   }
 
   test("/ (Divide) basic") {
@@ -99,6 +118,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(Divide(Literal(1.toShort), Literal(2.toShort)), 0.toShort)
     checkEvaluation(Divide(Literal(1), Literal(2)), 0)
     checkEvaluation(Divide(Literal(1.toLong), Literal(2.toLong)), 0.toLong)
+    checkEvaluation(Divide(positiveShortLit, negativeShortLit), 0.toShort)
+    checkEvaluation(Divide(positiveIntLit, negativeIntLit), 0)
+    checkEvaluation(Divide(positiveLongLit, negativeLongLit), 0L)
   }
 
   test("/ (Divide) for floating point") {
@@ -116,6 +138,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
       checkEvaluation(Remainder(left, Literal.create(null, right.dataType)), null)
       checkEvaluation(Remainder(left, Literal(convert(0))), null)  // mod by 0
     }
+    checkEvaluation(Remainder(positiveShortLit, positiveShortLit), 0.toShort)
+    checkEvaluation(Remainder(negativeShortLit, negativeShortLit), 0.toShort)
+    checkEvaluation(Remainder(positiveIntLit, positiveIntLit), 0)
+    checkEvaluation(Remainder(negativeIntLit, negativeIntLit), 0)
+    checkEvaluation(Remainder(positiveLongLit, positiveLongLit), 0L)
+    checkEvaluation(Remainder(negativeLongLit, negativeLongLit), 0L)
   }
 
   test("Abs") {
@@ -127,6 +155,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
       checkEvaluation(Abs(Literal(convert(-1))), convert(1))
       checkEvaluation(Abs(Literal.create(null, dataType)), null)
     }
+    checkEvaluation(Abs(positiveShortLit), positiveShort)
+    checkEvaluation(Abs(negativeShortLit), (- negativeShort).toShort)
+    checkEvaluation(Abs(positiveIntLit), positiveInt)
+    checkEvaluation(Abs(negativeIntLit), - negativeInt)
+    checkEvaluation(Abs(positiveLongLit), positiveLong)
+    checkEvaluation(Abs(negativeLongLit), - negativeLong)
   }
 
   test("MaxOf basic") {
@@ -138,6 +172,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
       checkEvaluation(MaxOf(Literal.create(null, small.dataType), large), convert(2))
       checkEvaluation(MaxOf(large, Literal.create(null, small.dataType)), convert(2))
     }
+    checkEvaluation(MaxOf(positiveShortLit, negativeShortLit), (positiveShort).toShort)
+    checkEvaluation(MaxOf(positiveIntLit, negativeIntLit), positiveInt)
+    checkEvaluation(MaxOf(positiveLongLit, negativeLongLit), positiveLong)
   }
 
   test("MaxOf for atomic type") {
@@ -156,6 +193,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
       checkEvaluation(MinOf(Literal.create(null, small.dataType), large), convert(2))
       checkEvaluation(MinOf(small, Literal.create(null, small.dataType)), convert(1))
     }
+    checkEvaluation(MinOf(positiveShortLit, negativeShortLit), (negativeShort).toShort)
+    checkEvaluation(MinOf(positiveIntLit, negativeIntLit), negativeInt)
+    checkEvaluation(MinOf(positiveLongLit, negativeLongLit), negativeLong)
   }
 
   test("MinOf for atomic type") {
@@ -174,9 +214,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
       checkEvaluation(Pmod(left, Literal.create(null, right.dataType)), null)
       checkEvaluation(Remainder(left, Literal(convert(0))), null)  // mod by 0
     }
-    checkEvaluation(Pmod(-7, 3), 2)
-    checkEvaluation(Pmod(7.2D, 4.1D), 3.1000000000000005)
-    checkEvaluation(Pmod(Decimal(0.7), Decimal(0.2)), Decimal(0.1))
-    checkEvaluation(Pmod(2L, Long.MaxValue), 2L)
+    checkEvaluation(Pmod(Literal(-7), Literal(3)), 2)
+    checkEvaluation(Pmod(Literal(7.2D), Literal(4.1D)), 3.1000000000000005)
+    checkEvaluation(Pmod(Literal(Decimal(0.7)), Literal(Decimal(0.2))), Decimal(0.1))
+    checkEvaluation(Pmod(Literal(2L), Literal(Long.MaxValue)), 2L)
+    checkEvaluation(Pmod(positiveShort, negativeShort), positiveShort.toShort)
+    checkEvaluation(Pmod(positiveInt, negativeInt), positiveInt)
+    checkEvaluation(Pmod(positiveLong, negativeLong), positiveLong)
   }
 }

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseFunctionsSuite.scala

@@ -23,6 +23,8 @@ import org.apache.spark.sql.types._
 
 class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
+  import IntegralLiteralTestUtils._
+
   test("BitwiseNOT") {
     def check(input: Any, expected: Any): Unit = {
       val expr = BitwiseNot(Literal(input))
@@ -37,6 +39,12 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     check(123456789123L, ~123456789123L)
 
     checkEvaluation(BitwiseNot(Literal.create(null, IntegerType)), null)
+    checkEvaluation(BitwiseNot(positiveShortLit), (~positiveShort).toShort)
+    checkEvaluation(BitwiseNot(negativeShortLit), (~negativeShort).toShort)
+    checkEvaluation(BitwiseNot(positiveIntLit), ~positiveInt)
+    checkEvaluation(BitwiseNot(negativeIntLit), ~negativeInt)
+    checkEvaluation(BitwiseNot(positiveLongLit), ~positiveLong)
+    checkEvaluation(BitwiseNot(negativeLongLit), ~negativeLong)
   }
 
   test("BitwiseAnd") {
@@ -56,6 +64,10 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(BitwiseAnd(nullLit, Literal(1)), null)
     checkEvaluation(BitwiseAnd(Literal(1), nullLit), null)
     checkEvaluation(BitwiseAnd(nullLit, nullLit), null)
+    checkEvaluation(BitwiseAnd(positiveShortLit, negativeShortLit),
+      (positiveShort & negativeShort).toShort)
+    checkEvaluation(BitwiseAnd(positiveIntLit, negativeIntLit), positiveInt & negativeInt)
+    checkEvaluation(BitwiseAnd(positiveLongLit, negativeLongLit), positiveLong & negativeLong)
   }
 
   test("BitwiseOr") {
@@ -75,6 +87,10 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(BitwiseOr(nullLit, Literal(1)), null)
     checkEvaluation(BitwiseOr(Literal(1), nullLit), null)
     checkEvaluation(BitwiseOr(nullLit, nullLit), null)
+    checkEvaluation(BitwiseOr(positiveShortLit, negativeShortLit),
+      (positiveShort | negativeShort).toShort)
+    checkEvaluation(BitwiseOr(positiveIntLit, negativeIntLit), positiveInt | negativeInt)
+    checkEvaluation(BitwiseOr(positiveLongLit, negativeLongLit), positiveLong | negativeLong)
   }
 
   test("BitwiseXor") {
@@ -94,5 +110,10 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(BitwiseXor(nullLit, Literal(1)), null)
     checkEvaluation(BitwiseXor(Literal(1), nullLit), null)
     checkEvaluation(BitwiseXor(nullLit, nullLit), null)
+
+    checkEvaluation(BitwiseXor(positiveShortLit, negativeShortLit),
+      (positiveShort ^ negativeShort).toShort)
+    checkEvaluation(BitwiseXor(positiveIntLit, negativeIntLit), positiveInt ^ negativeInt)
+    checkEvaluation(BitwiseXor(positiveLongLit, negativeLongLit), positiveLong ^ negativeLong)
   }
 }

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala

@@ -28,6 +28,8 @@ import org.apache.spark.unsafe.types.CalendarInterval
 
 class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
+  import IntegralLiteralTestUtils._
+
   val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
   val sdfDate = new SimpleDateFormat("yyyy-MM-dd")
   val d = new Date(sdf.parse("2015-04-08 13:10:15").getTime)
@@ -212,6 +214,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       null)
     checkEvaluation(DateAdd(Literal.create(null, DateType), Literal.create(null, IntegerType)),
       null)
+    checkEvaluation(
+      DateAdd(Literal(Date.valueOf("2016-02-28")), positiveIntLit), 49627)
+    checkEvaluation(
+      DateAdd(Literal(Date.valueOf("2016-02-28")), negativeIntLit), -15910)
   }
 
   test("date_sub") {
@@ -226,6 +232,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       null)
     checkEvaluation(DateSub(Literal.create(null, DateType), Literal.create(null, IntegerType)),
       null)
+    checkEvaluation(
+      DateSub(Literal(Date.valueOf("2016-02-28")), positiveIntLit), -15909)
+    checkEvaluation(
+      DateSub(Literal(Date.valueOf("2016-02-28")), negativeIntLit), 49628)
   }
 
   test("time_add") {
@@ -282,6 +292,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       null)
     checkEvaluation(
       AddMonths(Literal(Date.valueOf("2015-01-30")), Literal(Int.MinValue)), -7293498)
+    checkEvaluation(
+      AddMonths(Literal(Date.valueOf("2016-02-28")), positiveIntLit), 1014213)
+    checkEvaluation(
+      AddMonths(Literal(Date.valueOf("2016-02-28")), negativeIntLit), -980528)
   }
 
   test("months_between") {

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntegralLiteralTestUtils.scala

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+/**
+ * Utilities to make sure we pass the proper numeric ranges
+ */
+object IntegralLiteralTestUtils {
+
+  val positiveShort: Short = (Byte.MaxValue + 1).toShort
+  val negativeShort: Short = (Byte.MinValue - 1).toShort
+
+  val positiveShortLit: Literal = Literal(positiveShort)
+  val negativeShortLit: Literal = Literal(negativeShort)
+
+  val positiveInt: Int = Short.MaxValue + 1
+  val negativeInt: Int = Short.MinValue - 1
+
+  val positiveIntLit: Literal = Literal(positiveInt)
+  val negativeIntLit: Literal = Literal(negativeInt)
+
+  val positiveLong: Long = Int.MaxValue + 1L
+  val negativeLong: Long = Int.MinValue - 1L
+
+  val positiveLongLit: Literal = Literal(positiveLong)
+  val negativeLongLit: Literal = Literal(negativeLong)
+}

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathFunctionsSuite.scala

@@ -30,6 +30,8 @@ import org.apache.spark.sql.types._
 
 class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
+  import IntegralLiteralTestUtils._
+
   /**
    * Used for testing leaf math expressions.
    *
@@ -293,6 +295,9 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Bin(l3), java.lang.Long.toBinaryString(123), row)
     checkEvaluation(Bin(l4), java.lang.Long.toBinaryString(1234), row)
     checkEvaluation(Bin(l5), java.lang.Long.toBinaryString(-123), row)
+
+    checkEvaluation(Bin(positiveLongLit), java.lang.Long.toBinaryString(positiveLong))
+    checkEvaluation(Bin(negativeLongLit), java.lang.Long.toBinaryString(negativeLong))
   }
 
   test("log2") {
@@ -324,6 +329,15 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
     checkEvaluation(ShiftLeft(Literal(21.toLong), Literal(1)), 42.toLong)
     checkEvaluation(ShiftLeft(Literal(-21.toLong), Literal(1)), -42.toLong)
+
+    checkEvaluation(ShiftLeft(positiveIntLit, positiveIntLit), positiveInt << positiveInt)
+    checkEvaluation(ShiftLeft(positiveIntLit, negativeIntLit), positiveInt << negativeInt)
+    checkEvaluation(ShiftLeft(negativeIntLit, positiveIntLit), negativeInt << positiveInt)
+    checkEvaluation(ShiftLeft(negativeIntLit, negativeIntLit), negativeInt << negativeInt)
+    checkEvaluation(ShiftLeft(positiveLongLit, positiveIntLit), positiveLong << positiveInt)
+    checkEvaluation(ShiftLeft(positiveLongLit, negativeIntLit), positiveLong << negativeInt)
+    checkEvaluation(ShiftLeft(negativeLongLit, positiveIntLit), negativeLong << positiveInt)
+    checkEvaluation(ShiftLeft(negativeLongLit, negativeIntLit), negativeLong << negativeInt)
   }
 
   test("shift right") {
@@ -335,6 +349,15 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
     checkEvaluation(ShiftRight(Literal(42.toLong), Literal(1)), 21.toLong)
     checkEvaluation(ShiftRight(Literal(-42.toLong), Literal(1)), -21.toLong)
+
+    checkEvaluation(ShiftRight(positiveIntLit, positiveIntLit), positiveInt >> positiveInt)
+    checkEvaluation(ShiftRight(positiveIntLit, negativeIntLit), positiveInt >> negativeInt)
+    checkEvaluation(ShiftRight(negativeIntLit, positiveIntLit), negativeInt >> positiveInt)
+    checkEvaluation(ShiftRight(negativeIntLit, negativeIntLit), negativeInt >> negativeInt)
+    checkEvaluation(ShiftRight(positiveLongLit, positiveIntLit), positiveLong >> positiveInt)
+    checkEvaluation(ShiftRight(positiveLongLit, negativeIntLit), positiveLong >> negativeInt)
+    checkEvaluation(ShiftRight(negativeLongLit, positiveIntLit), negativeLong >> positiveInt)
+    checkEvaluation(ShiftRight(negativeLongLit, negativeIntLit), negativeLong >> negativeInt)
   }
 
   test("shift right unsigned") {
@@ -346,6 +369,23 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
     checkEvaluation(ShiftRightUnsigned(Literal(42.toLong), Literal(1)), 21.toLong)
     checkEvaluation(ShiftRightUnsigned(Literal(-42.toLong), Literal(1)), 9223372036854775787L)
+
+    checkEvaluation(ShiftRightUnsigned(positiveIntLit, positiveIntLit),
+      positiveInt >>> positiveInt)
+    checkEvaluation(ShiftRightUnsigned(positiveIntLit, negativeIntLit),
+      positiveInt >>> negativeInt)
+    checkEvaluation(ShiftRightUnsigned(negativeIntLit, positiveIntLit),
+      negativeInt >>> positiveInt)
+    checkEvaluation(ShiftRightUnsigned(negativeIntLit, negativeIntLit),
+      negativeInt >>> negativeInt)
+    checkEvaluation(ShiftRightUnsigned(positiveLongLit, positiveIntLit),
+      positiveLong >>> positiveInt)
+    checkEvaluation(ShiftRightUnsigned(positiveLongLit, negativeIntLit),
+      positiveLong >>> negativeInt)
+    checkEvaluation(ShiftRightUnsigned(negativeLongLit, positiveIntLit),
+      negativeLong >>> positiveInt)
+    checkEvaluation(ShiftRightUnsigned(negativeLongLit, negativeIntLit),
+      negativeLong >>> negativeInt)
   }
 
   test("hex") {