From df1b9f7b1a07bf8d806695a7684f9d69bf705093 Mon Sep 17 00:00:00 2001
From: Prashant Sharma <prashant.s@imaginea.com>
Date: Sat, 29 Mar 2014 23:03:03 -0700
Subject: [PATCH] SPARK-1336 Reducing the output of run-tests script.
Author: Prashant Sharma <prashant.s@imaginea.com>
Author: Prashant Sharma <scrapcodes@gmail.com>
Closes #262 from ScrapCodes/SPARK-1336/ReduceVerbosity and squashes the following commits:
87dfa54 [Prashant Sharma] Further reduction in noise and made pyspark tests to fail fast.
811170f [Prashant Sharma] Reducing the ouput of run-tests script.
---
.gitignore | 1 +
dev/run-tests | 7 +++----
dev/scalastyle | 27 +++++++++++++++++++++++++++
python/run-tests | 19 ++++++++++++-------
4 files changed, 43 insertions(+), 11 deletions(-)
create mode 100755 dev/scalastyle
diff --git a/.gitignore b/.gitignore
index 3a68abd955..cd9f90d559 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,3 +47,4 @@ spark-*-bin.tar.gz
unit-tests.log
/lib/
rat-results.txt
+scalastyle.txt
diff --git a/dev/run-tests b/dev/run-tests
index 6f115d2abd..a6fcc40a5b 100755
--- a/dev/run-tests
+++ b/dev/run-tests
@@ -39,18 +39,17 @@ JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*
echo "========================================================================="
echo "Running Apache RAT checks"
echo "========================================================================="
-
dev/check-license
echo "========================================================================="
echo "Running Scala style checks"
echo "========================================================================="
-sbt/sbt clean scalastyle
+dev/scalastyle
echo "========================================================================="
echo "Running Spark unit tests"
echo "========================================================================="
-sbt/sbt assembly test
+sbt/sbt assembly test | grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
echo "========================================================================="
echo "Running PySpark tests"
@@ -64,5 +63,5 @@ echo "========================================================================="
echo "Detecting binary incompatibilites with MiMa"
echo "========================================================================="
./bin/spark-class org.apache.spark.tools.GenerateMIMAIgnore
-sbt/sbt mima-report-binary-issues
+sbt/sbt mima-report-binary-issues | grep -v -e "info.*Resolving"
diff --git a/dev/scalastyle b/dev/scalastyle
new file mode 100755
index 0000000000..5a18f4d672
--- /dev/null
+++ b/dev/scalastyle
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+sbt/sbt clean scalastyle > scalastyle.txt
+ERRORS=$(cat scalastyle.txt | grep -e "error file")
+if test ! -z "$ERRORS"; then
+ echo -e "Scalastyle checks failed at following occurrences:\n$ERRORS"
+ exit 1
+else
+ echo -e "Scalastyle checks passed.\n"
+fi
diff --git a/python/run-tests b/python/run-tests
index a986ac9380..b2b60f08b4 100755
--- a/python/run-tests
+++ b/python/run-tests
@@ -29,8 +29,18 @@ FAILED=0
rm -f unit-tests.log
function run_test() {
- SPARK_TESTING=0 $FWDIR/bin/pyspark $1 2>&1 | tee -a unit-tests.log
+ SPARK_TESTING=0 $FWDIR/bin/pyspark $1 2>&1 | tee -a > unit-tests.log
FAILED=$((PIPESTATUS[0]||$FAILED))
+
+ # Fail and exit on the first test failure.
+ if [[ $FAILED != 0 ]]; then
+ cat unit-tests.log | grep -v "^[0-9][0-9]*" # filter all lines starting with a number.
+ echo -en "\033[31m" # Red
+ echo "Had test failures; see logs."
+ echo -en "\033[0m" # No color
+ exit -1
+ fi
+
}
run_test "pyspark/rdd.py"
@@ -46,12 +56,7 @@ run_test "pyspark/mllib/clustering.py"
run_test "pyspark/mllib/recommendation.py"
run_test "pyspark/mllib/regression.py"
-if [[ $FAILED != 0 ]]; then
- echo -en "\033[31m" # Red
- echo "Had test failures; see logs."
- echo -en "\033[0m" # No color
- exit -1
-else
+if [[ $FAILED == 0 ]]; then
echo -en "\033[32m" # Green
echo "Tests passed."
echo -en "\033[0m" # No color
--
GitLab