From f6301e4f0e79b471ddb523ad7a8c8dc0dd1e7e41 Mon Sep 17 00:00:00 2001
From: Yifan Zhao <yifanz16@illinois.edu>
Date: Wed, 17 Mar 2021 22:53:46 -0500
Subject: [PATCH] Use python for installer script for much, much better arg
handling
---
hpvm/install.sh | 13 +-
hpvm/llvm_patches/construct_patch.sh | 33 +---
hpvm/scripts/hpvm_installer.py | 246 ++++++++++++++++++++++++++
hpvm/scripts/llvm_installer.sh | 255 ---------------------------
4 files changed, 258 insertions(+), 289 deletions(-)
create mode 100755 hpvm/scripts/hpvm_installer.py
delete mode 100755 hpvm/scripts/llvm_installer.sh
diff --git a/hpvm/install.sh b/hpvm/install.sh
index 692c6195d2..dd737034f0 100755
--- a/hpvm/install.sh
+++ b/hpvm/install.sh
@@ -1,11 +1,6 @@
#!/bin/bash
-
-SCRIPTS_DIR=scripts
-
-BASH=/bin/bash
-
# Run installer script
-$BASH $SCRIPTS_DIR/llvm_installer.sh
-
-# Run the tests
-$BASH $SCRIPTS_DIR/automated_tests.sh
+# Pass on args to installer that can parse them
+scripts/hpvm_installer.py "$@"
+# Set path.
+export PATH=$BUILD_DIR/bin:$PATH
diff --git a/hpvm/llvm_patches/construct_patch.sh b/hpvm/llvm_patches/construct_patch.sh
index cc50fcc422..b957c853e7 100644
--- a/hpvm/llvm_patches/construct_patch.sh
+++ b/hpvm/llvm_patches/construct_patch.sh
@@ -1,29 +1,12 @@
#!/bin/sh
#### Computing Header Diff
-diff -u $LLVM_SRC_ROOT/include/llvm/Bitcode/LLVMBitCodes.h include/Bitcode/LLVMBitCodes.h > include/Bitcode/LLVMBitCodes.h.patch
-
-diff -u $LLVM_SRC_ROOT/include/llvm/IR/Attributes.td include/IR/Attributes.td > include/IR/Attributes.td.patch
-
-diff -u $LLVM_SRC_ROOT/include/llvm/IR/Intrinsics.td include/IR/Intrinsics.td > include/IR/Intrinsics.td.patch
-
-diff -u $LLVM_SRC_ROOT/include/llvm/Support/Debug.h include/Support/Debug.h > include/Support/Debug.h.patch
-
-
+for file in Bitcode/LLVMBitCodes.h IR/Attributes.td IR/Intrinsics.td Support/Debug.h; do
+ diff -u $LLVM_SRC_ROOT/include/llvm/$file include/$file > include/$file.patch || true
+done
#### Computing Source File Diff
-
-diff -u $LLVM_SRC_ROOT/lib/AsmParser/LLLexer.cpp lib/AsmParser/LLLexer.cpp > lib/AsmParser/LLLexer.cpp.patch
-
-diff -u $LLVM_SRC_ROOT/lib/AsmParser/LLLexer.h lib/AsmParser/LLLexer.h > lib/AsmParser/LLLexer.h.patch
-
-diff -u $LLVM_SRC_ROOT/lib/AsmParser/LLParser.cpp lib/AsmParser/LLParser.cpp > lib/AsmParser/LLParser.cpp.patch
-
-diff -u $LLVM_SRC_ROOT/lib/AsmParser/LLParser.h lib/AsmParser/LLParser.h > lib/AsmParser/LLParser.h.patch
-
-diff -u $LLVM_SRC_ROOT/lib/AsmParser/LLToken.h lib/AsmParser/LLToken.h > lib/AsmParser/LLToken.h.patch
-
-diff -u $LLVM_SRC_ROOT/lib/IR/Attributes.cpp lib/IR/Attributes.cpp > lib/IR/Attributes.cpp.patch
-
-diff -u $LLVM_SRC_ROOT/lib/Bitcode/Reader/BitcodeReader.cpp lib/Bitcode/Reader/BitcodeReader.cpp > lib/Bitcode/Reader/BitcodeReader.cpp.patch
-
-diff -u $LLVM_SRC_ROOT/lib/Bitcode/Writer/BitcodeWriter.cpp lib/Bitcode/Writer/BitcodeWriter.cpp > lib/Bitcode/Writer/BitcodeWriter.cpp.patch
+for file in AsmParser/LLLexer.cpp AsmParser/LLLexer.h AsmParser/LLParser.cpp \
+ AsmParser/LLParser.h AsmParser/LLToken.h IR/Attributes.cpp \
+ Bitcode/Reader/BitcodeReader.cpp Bitcode/Writer/BitcodeWriter.cpp; do
+ diff -u $LLVM_SRC_ROOT/lib/$file lib/$file > lib/$file.patch || true
+done
diff --git a/hpvm/scripts/hpvm_installer.py b/hpvm/scripts/hpvm_installer.py
new file mode 100755
index 0000000000..bf879cdb1d
--- /dev/null
+++ b/hpvm/scripts/hpvm_installer.py
@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+from pathlib import Path
+from argparse import ArgumentParser, Namespace
+from subprocess import check_call
+from os import makedirs, chdir, environ
+
+VERSION = "9.0.0"
+URL = "http://releases.llvm.org"
+WGET = "wget"
+CLANG_DIR = f"cfe-{VERSION}.src"
+CLANG_TARBALL = f"{CLANG_DIR}.tar.xz"
+LLVM_DIR = f"llvm-{VERSION}.src"
+LLVM_TARBALL = f"{LLVM_DIR}.tar.xz"
+
+ROOT_DIR = Path.cwd()
+BUILD_DIR = ROOT_DIR / "build"
+TEST_DIR = ROOT_DIR / "test"
+LLVM_LIT = BUILD_DIR / "bin/llvm-lit"
+
+LINKS = [
+ "CMakeLists.txt",
+ "cmake",
+ "include",
+ "lib",
+ "projects",
+ "test",
+ "tools",
+]
+MAKE_TARGETS = ["approxhpvm.py"]
+MAKE_TEST_TARGETS = ["hpvm-check"]
+
+
+def parse_args():
+ parser = ArgumentParser(
+ "hpvm_installer", description="Script for automatic HPVM installation."
+ )
+ parser.add_argument(
+ "-m",
+ "--no-build",
+ action="store_true",
+ help="Configure but don't build HPVM. "
+ "This will require you to install HPVM manually using cmake and make. "
+ "For more details, refer to README.md. Default: False.",
+ )
+ parser.add_argument(
+ "-j",
+ "--parallel",
+ type=int,
+ default=2,
+ help="How many threads to build with. This argument is relayed on to 'make'. Default: 2",
+ )
+ parser.add_argument(
+ "-t",
+ "--targets",
+ type=str,
+ default="all",
+ help="Build target(s) for LLVM such as X86, ARM. "
+ 'Use semicolon to separate multiple targets such as "X86;ARM". '
+ 'Defaults to "all" which is to build all supported targets. '
+ "Supported targets: AArch64, AMDGPU, ARM, BPF, Hexagon, Mips, MSP430, NVPTX, PowerPC, "
+ "Sparc, SystemZ, X86, XCore.",
+ )
+ parser.add_argument(
+ "-r", "--run-tests", action="store_true", help="Build and run test cases"
+ )
+ return parser.parse_args()
+
+
+def prompt_args():
+ def parse_yn(s: str):
+ table = {"y": True, "n": False}
+ return table.get(s)
+
+ def parse_int(s: str):
+ try:
+ v = int(s)
+ except ValueError:
+ return None
+ if v <= 0:
+ return None
+
+ def parse_targets(s: str):
+ if " " in s:
+ return None
+ return s
+
+ print("No Flags found. Using command line prompts.")
+ print("Alternatively, please call this script with -h for all available options.")
+ auto_build = input_with_check(
+ "Build and install HPVM automatically? [y/n]: ", parse_yn, "Please enter y or n"
+ )
+ if not auto_build:
+ # Just stuff something in the other fields. We won't need them.
+ return Namespace(no_build=True, parallel="", targets="", run_tests=False)
+ threads = input_with_check(
+ "Number of threads: ", parse_int, "Please enter a positive integer"
+ )
+ print(
+ "These build targets are supported: AArch64, AMDGPU, ARM, BPF, Hexagon, "
+ "Mips, MSP430, NVPTX, PowerPC, Sparc, SystemZ, X86, XCore.\n"
+ "If building for multiple targets, seperate options with semicolon:\n"
+ "e.g. X86;ARM"
+ )
+ targets = input_with_check(
+ "Build target: ", parse_targets, "Input shouldn't contain space"
+ )
+
+ run_tests = input_with_check(
+ "Build and run tests? [y/n]: ", parse_yn, "Please enter y or n"
+ )
+ return Namespace(
+ no_build=not auto_build, parallel=threads, targets=targets, run_tests=run_tests
+ )
+
+
+def print_args(args):
+ print("Running with the following options:")
+ print(f" Automated: {not args.no_build}")
+ print(f" Threads: {args.parallel}")
+ print(f" Targets: {args.targets}")
+ print(f" Run tests: {args.run_tests}")
+
+
+def check_download_llvm_clang():
+ if Path("llvm/").is_dir():
+ print("Found LLVM, not extracting it again.")
+ else:
+ if Path(LLVM_TARBALL).is_file():
+ print(f"Found {LLVM_TARBALL}, not downloading it again.")
+ else:
+ print(f"Downloading {LLVM_TARBALL}...")
+ print(f"=============================")
+ check_call([WGET, f"{URL}/{VERSION}/{LLVM_TARBALL}"])
+ check_call(["tar", "xf", LLVM_TARBALL])
+ check_call(["mv", LLVM_DIR, "llvm"])
+ tools = Path("llvm/tools")
+ assert tools.is_dir(), "Problem with LLVM download. Exiting!"
+ if Path(LLVM_TARBALL).is_file():
+ Path(LLVM_TARBALL).unlink() # Remove tarball
+ environ["LLVM_SRC_ROOT"] = str(ROOT_DIR / "llvm")
+
+ if (tools / "clang/").is_dir():
+ print("Found clang, not downloading it again.")
+ return
+ chdir(tools)
+ print(f"Downloading {CLANG_TARBALL}...")
+ print(f"=============================")
+ check_call([WGET, f"{URL}/{VERSION}/{CLANG_TARBALL}"])
+ check_call(["tar", "xf", CLANG_TARBALL])
+ check_call(["mv", CLANG_DIR, "clang"])
+ assert Path("clang/").is_dir(), "Problem with clang download. Exiting!"
+ if Path(CLANG_TARBALL).is_file():
+ Path(CLANG_TARBALL).unlink()
+ chdir(ROOT_DIR)
+
+
+def link_and_patch():
+ from os import symlink
+
+ hpvm = ROOT_DIR / "llvm/tools/hpvm"
+ print("Adding HPVM sources to tree...")
+ makedirs(hpvm, exist_ok=True)
+ for link in LINKS:
+ if not (hpvm / link).exists():
+ print(ROOT_DIR / link, hpvm / link)
+ symlink(ROOT_DIR / link, hpvm / link)
+ print("Applying HPVM patches...")
+ chdir("llvm_patches")
+ check_call(["bash", "./construct_patch.sh"])
+ check_call(["bash", "./apply_patch.sh"])
+ print("Patches applied.")
+ chdir("..")
+
+
+def maybe_build(build: bool, nthreads: int, targets: str, build_test_targets: bool):
+ if not build:
+ print(
+ """
+HPVM not installed.
+To complete installation, follow these instructions:
+- Create and navigate to a folder "./build"
+- Run "cmake ../llvm [options]". Find potential options in README.md.
+- Run "make -j<number of threads> approxhpvm.py" and then "make install"
+For more details refer to README.md.
+"""
+ )
+ return
+ print("Now building...")
+ print(f"Using {nthreads} threads to build HPVM.")
+ makedirs(BUILD_DIR, exist_ok=True)
+
+ chdir(BUILD_DIR)
+ cmake_args = [
+ "cmake",
+ "../llvm",
+ "-DCMAKE_C_COMPILER=gcc",
+ "-DCMAKE_CXX_COMPILER=g++",
+ f"-DLLVM_TARGETS_TO_BUILD={targets}",
+ ]
+ print(f"CMake: {' '.join(cmake_args)}")
+ print(f"=============================")
+ check_call(cmake_args)
+ make_args = ["make", f"-j{nthreads}", *MAKE_TARGETS]
+ if build_test_targets:
+ make_args += MAKE_TEST_TARGETS
+ print(f"Make: {' '.join(make_args)}")
+ print(f"=============================")
+ check_call(make_args)
+ chdir(ROOT_DIR)
+
+
+def run_tests():
+ chdir(BUILD_DIR)
+ # Run regression tests
+ check_call([LLVM_LIT, "-v", TEST_DIR / "regressionTests"])
+ # Run unit tests
+ check_call([LLVM_LIT, "-v", TEST_DIR / "unitTests"])
+
+
+def input_with_check(prompt: str, parse, prompt_when_invalid: str):
+ input_str = input(prompt)
+ value = parse(input_str)
+ while value is None:
+ print(f"{prompt_when_invalid}; got {input_str}")
+ input_str = input(prompt)
+ value = parse(input_str)
+ return value
+
+
+def main():
+ from sys import argv
+
+ # Don't parse args if no args given -- use prompt mode
+ args = prompt_args() if len(argv) == 1 else parse_args()
+ print_args(args)
+ check_download_llvm_clang()
+ link_and_patch()
+ maybe_build(not args.no_build, args.parallel, args.targets, args.run_tests)
+ if args.run_tests:
+ run_tests()
+ else:
+ print("Skipping tests.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/hpvm/scripts/llvm_installer.sh b/hpvm/scripts/llvm_installer.sh
deleted file mode 100755
index a8fa022047..0000000000
--- a/hpvm/scripts/llvm_installer.sh
+++ /dev/null
@@ -1,255 +0,0 @@
-#!/bin/bash
-
-function read_yn {
- read -p "$1 [y/n]: " read_value
- while [ ! $read_value == "y" ] && [ ! $read_value == "n" ]; do
- echo "Please answer y or n; got $read_value"
- read -p "$1 [y/n]:" read_value
- done
- eval $2=$read_value
-}
-
-VERSION="9.0.0"
-
-URL="http://releases.llvm.org"
-
-WGET=wget
-
-CURRENT_DIR=`pwd`
-INSTALL_DIR=`pwd`/install
-BUILD_DIR=$CURRENT_DIR/build
-
-# Using 2 threads by default
-NUM_THREADS=2
-
-SUFFIX=".tar.xz"
-CLANG_SRC="cfe-$VERSION.src"
-LLVM_SRC="llvm-$VERSION.src"
-
-HPVM_RT=hpvm-rt/hpvm-rt.bc
-
-
-TARGET=all
-TARGET_INPUT=all
-FLAGGED=false
-
-# Get flags
-while getopts 'hmj:t:' opt; do
- case $opt in
- h)
- echo
- echo
- echo "This is the help menu for HPVM installation"
- echo
- echo "There are 3 options for installation:"
- echo
- echo "-m is a manual installation flag. This will require you to install HPVM manually by running cmake and make manually."
- echo "For more details, refer to README.md. Defaults to automatic installation."
- echo
- echo "-j is the threads flag. Accepts one argument: how many threads to build with."
- echo "To build with 2 threads, enter -j2. Defaults to 2 threads."
- echo
- echo "-t is the build target flag. Accepts one argument: which build target(s) you would like to build to."
- echo "For single target, enter -a ARM. For multiple targets, enter -t \"X86;ARM\"."
- echo "Supports the following targets: AArch64, AMDGPU, ARM, BPF, Hexagon, Mips, MSP430, NVPTX, PowerPC, Sparc, SystemZ, X86, XCore."
- echo "Defaults to targeting all supported architectures."
- echo
- echo "If no flags are provided, the script will use command line prompts for all options."
- echo
- exit
- ;;
- m)
- AUTOMATE=false
- FLAGGED=true
- ;;
- j)
- if ! [[ $OPTARG =~ ^[0-9]+$ ]]; then
- echo "Invalid argument for # of threads: $OPTARG"
- exit -1;
- else
- NUM_THREADS=$OPTARG
- FLAGGED=true
- fi
- ;;
- t)
- TARGET=$OPTARG
- FLAGGED=true
- ;;
- esac
-done
-
-if $FLAGGED; then
- echo "Running with the following options:"
- echo Automated: $AUTOMATE
- echo Threads: $NUM_THREADS
- echo Targets: $TARGET
- echo
-else
- echo "No Flags found. Using command line prompts."
- read -p "Build and install HPVM automatically? (y or n): " AUTOMATE_INPUT
-
- if [[ $AUTOMATE_INPUT == "" ]]; then
- echo "No input given. Using default: $AUTOMATE"
- elif [[ ! $AUTOMATE_INPUT == "y" ]] && [[ ! $AUTOMATE_INPUT == "n" ]]; then
- echo "Invalid input. Using default: $AUTOMATE"
- elif [[ $AUTOMATE_INPUT == "n" ]]; then
- AUTOMATE=false
- fi
-
-
- echo
- read -p "Number of threads: " NUM_THREADS_INPUT
-
- if [[ $NUM_THREADS_INPUT == "" ]]; then
- echo "No input given. Using default: $NUM_THREADS"
- elif ! [[ $NUM_THREADS_INPUT =~ ^[0-9]+$ ]]; then
- echo "Given input is not an integer. Using default: $NUM_THREADS"
- elif [ ! $NUM_THREADS_INPUT -gt 0 ]; then
- echo "Given input is not greater than 0. Using default: $NUM_THREADS"
- else
- NUM_THREADS=$NUM_THREADS_INPUT
- fi
-
- echo
- echo
- echo "Supports the following options: AArch64, AMDGPU, ARM, BPF, Hexagon, Mips, MSP430, NVPTX, PowerPC, Sparc, SystemZ, X86, XCore."
- echo "If building for multiple targets, seperate options with semicolon:"
- echo "e.g. X86;ARM"
- read -p "Build target: " TARGET_INPUT
- if [[ $TARGET_INPUT == "" ]]; then
- echo "No input given. Using default: $TARGET"
- else
- TARGET=$TARGET_INPUT
- fi
- echo
-
- echo "Running with the following options:"
- echo Automated: $AUTOMATE
- echo Threads: $NUM_THREADS
- echo Targets: $TARGET
- echo
-fi
-
-if [ -d $LLVM_SRC ]; then
- echo Found $LLVM_SRC, not dowloading it again!
-elif [ -d llvm ]; then
- echo Found LLVM, not downloading it again!
-else
- echo $WGET $URL/$VERSION/$LLVM_SRC$SUFFIX
- $WGET $URL/$VERSION/$LLVM_SRC$SUFFIX
- tar xf $LLVM_SRC$SUFFIX
- rm $LLVM_SRC$SUFFIX
-fi
-
-if [ -d $LLVM_SRC ]; then
- echo Everything looks sane.
- mv $LLVM_SRC llvm
-elif [ -d llvm ]; then
- echo Everything looks sane.
-else
- echo Problem with LLVM download. Exiting!
- exit
-fi
-
-LLVM_SRC=llvm
-
-if [ -d $CURRENT_DIR/$LLVM_SRC/tools ]; then
- cd $CURRENT_DIR/$LLVM_SRC/tools
- echo In tools.
-else
- echo Something is wrong with LLVM checkout. Exiting!
- exit 1
-fi
-
-if [ -d clang ]; then
- echo Found clang! Not downloading clang again.
-else
- $WGET $URL/$VERSION/$CLANG_SRC$SUFFIX
- tar xf $CLANG_SRC$SUFFIX
- rm $CLANG_SRC$SUFFIX
- mv $CLANG_SRC clang
- if [ -d clang ]; then
- echo Everything looks sane.
- else
- echo Problem with clang download. Exiting!
- exit
- fi
-fi
-
-cd $CURRENT_DIR
-
-HPVM_DIR=$CURRENT_DIR/$LLVM_SRC/tools/hpvm
-
-if [ ! -d $HPVM_DIR ]; then
- echo Adding HPVM sources to tree
- mkdir -p $HPVM_DIR
- ln -s $CURRENT_DIR/CMakeLists.txt $HPVM_DIR
- ln -s $CURRENT_DIR/cmake $HPVM_DIR/
- ln -s $CURRENT_DIR/include $HPVM_DIR/
- ln -s $CURRENT_DIR/lib $HPVM_DIR/
- ln -s $CURRENT_DIR/projects $HPVM_DIR/
- ln -s $CURRENT_DIR/test $HPVM_DIR/
- ln -s $CURRENT_DIR/tools $HPVM_DIR/
-else
- echo $CURRENT_DIR/$LLVM_SRC/tools/hpvm exists.
-fi
-
-export LLVM_SRC_ROOT=$CURRENT_DIR/$LLVM_SRC
-
-echo Applying HPVM patches
-cd $CURRENT_DIR/llvm_patches
-/bin/bash ./construct_patch.sh
-/bin/bash ./apply_patch.sh
-
-echo Patches applied.
-
-if ! $AUTOMATE ; then
- echo
- echo "HPVM not installed."
- echo "To complete installation, follow these instructions:"
- echo " - Create and navigate to a folder \"./build\" "
- echo " - Run \"cmake ../llvm [options]\". Find potential options in README.md."
- echo " - Run \"make -j<number of threads> approxhpvm.py\" and then \"make install\""
- echo "For more details refer to README.md."
- echo
- echo "Exiting."
- exit
-fi
-
-echo
-echo Now building...
-
-echo Using $NUM_THREADS threads to build HPVM.
-echo
-
-cd $CURRENT_DIR
-
-if [ ! -d $BUILD_DIR ]; then
- mkdir -p $BUILD_DIR
-fi
-
-if [ ! -d $INSTALL_DIR ]; then
- mkdir -p $INSTALL_DIR
-fi
-
-export PATH=$BUILD_DIR/bin:$PATH
-
-cd $BUILD_DIR
-echo cmake ../$LLVM_SRC -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DLLVM_TARGETS_TO_BUILD=$TARGET -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR
-cmake ../$LLVM_SRC -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DLLVM_TARGETS_TO_BUILD=$TARGET -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR
-
-echo make -j$NUM_THREADS approxhpvm.py
-make -j$NUM_THREADS approxhpvm.py
-#make install
-
-if [ -f $BUILD_DIR/tools/hpvm/projects/$HPVM_RT ]; then
- true
-else
- echo $BUILD_DIR/tools/hpvm/projects/$HPVM_RT
- echo HPVM not installed properly.
- exit 0
-fi
-
-cd $CURRENT_DIR
-
-
--
GitLab