From 314cf51ded52834cfbaacf58d3d05a220965ca2a Mon Sep 17 00:00:00 2001
From: Josh Rosen <joshrosen@databricks.com>
Date: Mon, 27 Mar 2017 10:23:28 -0700
Subject: [PATCH] [SPARK-20102] Fix nightly packaging and RC packaging scripts
w/ two minor build fixes
## What changes were proposed in this pull request?
The master snapshot publisher builds are currently broken due to two minor build issues:
1. For unknown reasons, the LFTP `mkdir -p` command began throwing errors when the remote directory already exists. This change of behavior might have been caused by configuration changes in the ASF's SFTP server, but I'm not entirely sure of that. To work around this problem, this patch updates the script to ignore errors from the `lftp mkdir -p` commands.
2. The PySpark `setup.py` file references a non-existent `pyspark.ml.stat` module, causing Python packaging to fail by complaining about a missing directory. The fix is to simply drop that line from the setup script.
## How was this patch tested?
The LFTP fix was tested by manually running the failing commands on AMPLab Jenkins against the ASF SFTP server. The PySpark fix was tested locally.
Author: Josh Rosen <joshrosen@databricks.com>
Closes #17437 from JoshRosen/spark-20102.
---
dev/create-release/release-build.sh | 8 ++++----
python/setup.py | 1 -
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index e1db997a7d..7976d8a039 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -246,7 +246,7 @@ if [[ "$1" == "package" ]]; then
dest_dir="$REMOTE_PARENT_DIR/${DEST_DIR_NAME}-bin"
echo "Copying release tarballs to $dest_dir"
# Put to new directory:
- LFTP mkdir -p $dest_dir
+ LFTP mkdir -p $dest_dir || true
LFTP mput -O $dest_dir 'spark-*'
LFTP mput -O $dest_dir 'pyspark-*'
LFTP mput -O $dest_dir 'SparkR_*'
@@ -254,7 +254,7 @@ if [[ "$1" == "package" ]]; then
LFTP "rm -r -f $REMOTE_PARENT_DIR/latest || exit 0"
LFTP mv $dest_dir "$REMOTE_PARENT_DIR/latest"
# Re-upload a second time and leave the files in the timestamped upload directory:
- LFTP mkdir -p $dest_dir
+ LFTP mkdir -p $dest_dir || true
LFTP mput -O $dest_dir 'spark-*'
LFTP mput -O $dest_dir 'pyspark-*'
LFTP mput -O $dest_dir 'SparkR_*'
@@ -271,13 +271,13 @@ if [[ "$1" == "docs" ]]; then
PRODUCTION=1 RELEASE_VERSION="$SPARK_VERSION" jekyll build
echo "Copying release documentation to $dest_dir"
# Put to new directory:
- LFTP mkdir -p $dest_dir
+ LFTP mkdir -p $dest_dir || true
LFTP mirror -R _site $dest_dir
# Delete /latest directory and rename new upload to /latest
LFTP "rm -r -f $REMOTE_PARENT_DIR/latest || exit 0"
LFTP mv $dest_dir "$REMOTE_PARENT_DIR/latest"
# Re-upload a second time and leave the files in the timestamped upload directory:
- LFTP mkdir -p $dest_dir
+ LFTP mkdir -p $dest_dir || true
LFTP mirror -R _site $dest_dir
cd ..
exit 0
diff --git a/python/setup.py b/python/setup.py
index 47eab98e0f..f50035435e 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -167,7 +167,6 @@ try:
'pyspark.ml',
'pyspark.ml.linalg',
'pyspark.ml.param',
- 'pyspark.ml.stat',
'pyspark.sql',
'pyspark.streaming',
'pyspark.bin',
--
GitLab