Skip to content
Snippets Groups Projects
Commit d61f2c15 authored by WangTaoTheTonic's avatar WangTaoTheTonic Committed by Michael Armbrust
Browse files

[SPARK-3658][SQL] Start thrift server as a daemon

https://issues.apache.org/jira/browse/SPARK-3658

And keep the `CLASS_NOT_FOUND_EXIT_STATUS` and exit message in `SparkSubmit.scala`.

Author: WangTaoTheTonic <barneystinson@aliyun.com>
Author: WangTao <barneystinson@aliyun.com>

Closes #2509 from WangTaoTheTonic/thriftserver and squashes the following commits:

5dcaab2 [WangTaoTheTonic] issue about coupling
8ad9f95 [WangTaoTheTonic] generalization
598e21e [WangTao] take thrift server as a daemon
parent fcad3fae
No related branches found
No related tags found
No related merge requests found
...@@ -24,7 +24,6 @@ ...@@ -24,7 +24,6 @@
set -o posix set -o posix
CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver"
CLASS_NOT_FOUND_EXIT_STATUS=101
# Figure out where Spark is installed # Figure out where Spark is installed
FWDIR="$(cd "`dirname "$0"`"/..; pwd)" FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
...@@ -53,13 +52,4 @@ source "$FWDIR"/bin/utils.sh ...@@ -53,13 +52,4 @@ source "$FWDIR"/bin/utils.sh
SUBMIT_USAGE_FUNCTION=usage SUBMIT_USAGE_FUNCTION=usage
gatherSparkSubmitOpts "$@" gatherSparkSubmitOpts "$@"
"$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" exec "$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}"
exit_status=$?
if [[ exit_status -eq CLASS_NOT_FOUND_EXIT_STATUS ]]; then
echo
echo "Failed to load Spark SQL CLI main class $CLASS."
echo "You need to build Spark with -Phive."
fi
exit $exit_status
...@@ -320,6 +320,10 @@ object SparkSubmit { ...@@ -320,6 +320,10 @@ object SparkSubmit {
} catch { } catch {
case e: ClassNotFoundException => case e: ClassNotFoundException =>
e.printStackTrace(printStream) e.printStackTrace(printStream)
if (childMainClass.contains("thriftserver")) {
println(s"Failed to load main class $childMainClass.")
println("You need to build Spark with -Phive.")
}
System.exit(CLASS_NOT_FOUND_EXIT_STATUS) System.exit(CLASS_NOT_FOUND_EXIT_STATUS)
} }
......
...@@ -62,7 +62,7 @@ then ...@@ -62,7 +62,7 @@ then
shift shift
fi fi
startStop=$1 option=$1
shift shift
command=$1 command=$1
shift shift
...@@ -122,9 +122,9 @@ if [ "$SPARK_NICENESS" = "" ]; then ...@@ -122,9 +122,9 @@ if [ "$SPARK_NICENESS" = "" ]; then
fi fi
case $startStop in case $option in
(start) (start|spark-submit)
mkdir -p "$SPARK_PID_DIR" mkdir -p "$SPARK_PID_DIR"
...@@ -142,8 +142,14 @@ case $startStop in ...@@ -142,8 +142,14 @@ case $startStop in
spark_rotate_log "$log" spark_rotate_log "$log"
echo starting $command, logging to $log echo starting $command, logging to $log
cd "$SPARK_PREFIX" if [ $option == spark-submit ]; then
nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & source "$SPARK_HOME"/bin/utils.sh
gatherSparkSubmitOpts "$@"
nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-submit --class $command \
"${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" >> "$log" 2>&1 < /dev/null &
else
nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null &
fi
newpid=$! newpid=$!
echo $newpid > $pid echo $newpid > $pid
sleep 2 sleep 2
......
...@@ -27,7 +27,6 @@ set -o posix ...@@ -27,7 +27,6 @@ set -o posix
FWDIR="$(cd "`dirname "$0"`"/..; pwd)" FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
CLASS="org.apache.spark.sql.hive.thriftserver.HiveThriftServer2" CLASS="org.apache.spark.sql.hive.thriftserver.HiveThriftServer2"
CLASS_NOT_FOUND_EXIT_STATUS=101
function usage { function usage {
echo "Usage: ./sbin/start-thriftserver [options] [thrift server options]" echo "Usage: ./sbin/start-thriftserver [options] [thrift server options]"
...@@ -49,17 +48,6 @@ if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then ...@@ -49,17 +48,6 @@ if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
exit 0 exit 0
fi fi
source "$FWDIR"/bin/utils.sh export SUBMIT_USAGE_FUNCTION=usage
SUBMIT_USAGE_FUNCTION=usage
gatherSparkSubmitOpts "$@"
"$FWDIR"/bin/spark-submit --class $CLASS "${SUBMISSION_OPTS[@]}" spark-internal "${APPLICATION_OPTS[@]}" exec "$FWDIR"/sbin/spark-daemon.sh spark-submit $CLASS 1 "$@"
exit_status=$?
if [[ exit_status -eq CLASS_NOT_FOUND_EXIT_STATUS ]]; then
echo
echo "Failed to load Hive Thrift server main class $CLASS."
echo "You need to build Spark with -Phive."
fi
exit $exit_status
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Stops the thrift server on the machine this script is executed on.
sbin="`dirname "$0"`"
sbin="`cd "$sbin"; pwd`"
"$sbin"/spark-daemon.sh stop org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment