Use python for installer script for much, much better arg handling

f6301e4f · Yifan Zhao · 87284f7b · f6301e4f · f6301e4f · f6301e4f
Commit f6301e4f authored 4 years ago by Yifan Zhao
--- a/hpvm/install.sh
+++ b/hpvm/install.sh
 #!/bin/bash
-SCRIPTS_DIR=scripts
-BASH=/bin/bash
 # Run installer script
-$BASH $SCRIPTS_DIR/llvm_installer.sh
+# Pass on args to installer that can parse them
+scripts/hpvm_installer.py "$@"
-# Run the tests
+# Set path.
-$BASH $SCRIPTS_DIR/automated_tests.sh
+export PATH=$BUILD_DIR/bin:$PATH
--- a/hpvm/llvm_patches/construct_patch.sh
+++ b/hpvm/llvm_patches/construct_patch.sh
 #!/bin/sh
 #### Computing Header Diff
-diff -u  $LLVM_SRC_ROOT/include/llvm/Bitcode/LLVMBitCodes.h  include/Bitcode/LLVMBitCodes.h > include/Bitcode/LLVMBitCodes.h.patch 
+for file in Bitcode/LLVMBitCodes.h IR/Attributes.td IR/Intrinsics.td Support/Debug.h; do
+    diff -u $LLVM_SRC_ROOT/include/llvm/$file include/$file > include/$file.patch || true
-diff -u  $LLVM_SRC_ROOT/include/llvm/IR/Attributes.td   include/IR/Attributes.td   > include/IR/Attributes.td.patch
+done
-diff -u  $LLVM_SRC_ROOT/include/llvm/IR/Intrinsics.td   include/IR/Intrinsics.td > include/IR/Intrinsics.td.patch
-diff -u  $LLVM_SRC_ROOT/include/llvm/Support/Debug.h   include/Support/Debug.h > include/Support/Debug.h.patch
 #### Computing Source File Diff
+for file in AsmParser/LLLexer.cpp AsmParser/LLLexer.h AsmParser/LLParser.cpp \
-diff -u  $LLVM_SRC_ROOT/lib/AsmParser/LLLexer.cpp   lib/AsmParser/LLLexer.cpp > lib/AsmParser/LLLexer.cpp.patch 
+            AsmParser/LLParser.h AsmParser/LLToken.h IR/Attributes.cpp \
+            Bitcode/Reader/BitcodeReader.cpp Bitcode/Writer/BitcodeWriter.cpp; do
-diff -u  $LLVM_SRC_ROOT/lib/AsmParser/LLLexer.h   lib/AsmParser/LLLexer.h > lib/AsmParser/LLLexer.h.patch
+    diff -u $LLVM_SRC_ROOT/lib/$file lib/$file > lib/$file.patch || true
+done
-diff -u  $LLVM_SRC_ROOT/lib/AsmParser/LLParser.cpp   lib/AsmParser/LLParser.cpp > lib/AsmParser/LLParser.cpp.patch
-diff -u  $LLVM_SRC_ROOT/lib/AsmParser/LLParser.h   lib/AsmParser/LLParser.h > lib/AsmParser/LLParser.h.patch
-diff -u  $LLVM_SRC_ROOT/lib/AsmParser/LLToken.h   lib/AsmParser/LLToken.h > lib/AsmParser/LLToken.h.patch
-diff -u  $LLVM_SRC_ROOT/lib/IR/Attributes.cpp   lib/IR/Attributes.cpp > lib/IR/Attributes.cpp.patch
-diff -u  $LLVM_SRC_ROOT/lib/Bitcode/Reader/BitcodeReader.cpp   lib/Bitcode/Reader/BitcodeReader.cpp > lib/Bitcode/Reader/BitcodeReader.cpp.patch
-diff -u  $LLVM_SRC_ROOT/lib/Bitcode/Writer/BitcodeWriter.cpp   lib/Bitcode/Writer/BitcodeWriter.cpp > lib/Bitcode/Writer/BitcodeWriter.cpp.patch
--- a/hpvm/scripts/hpvm_installer.py
+++ b/hpvm/scripts/hpvm_installer.py
+#!/usr/bin/env python3
+from pathlib import Path
+from argparse import ArgumentParser, Namespace
+from subprocess import check_call
+from os import makedirs, chdir, environ
+VERSION = "9.0.0"
+URL = "http://releases.llvm.org"
+WGET = "wget"
+CLANG_DIR = f"cfe-{VERSION}.src"
+CLANG_TARBALL = f"{CLANG_DIR}.tar.xz"
+LLVM_DIR = f"llvm-{VERSION}.src"
+LLVM_TARBALL = f"{LLVM_DIR}.tar.xz"
+ROOT_DIR = Path.cwd()
+BUILD_DIR = ROOT_DIR / "build"
+TEST_DIR = ROOT_DIR / "test"
+LLVM_LIT = BUILD_DIR / "bin/llvm-lit"
+LINKS = [
+    "CMakeLists.txt",
+    "cmake",
+    "include",
+    "lib",
+    "projects",
+    "test",
+    "tools",
+]
+MAKE_TARGETS = ["approxhpvm.py"]
+MAKE_TEST_TARGETS = ["hpvm-check"]
+def parse_args():
+    parser = ArgumentParser(
+        "hpvm_installer", description="Script for automatic HPVM installation."
+    )
+    parser.add_argument(
+        "-m",
+        "--no-build",
+        action="store_true",
+        help="Configure but don't build HPVM. "
+        "This will require you to install HPVM manually using cmake and make. "
+        "For more details, refer to README.md. Default: False.",
+    )
+    parser.add_argument(
+        "-j",
+        "--parallel",
+        type=int,
+        default=2,
+        help="How many threads to build with. This argument is relayed on to 'make'. Default: 2",
+    )
+    parser.add_argument(
+        "-t",
+        "--targets",
+        type=str,
+        default="all",
+        help="Build target(s) for LLVM such as X86, ARM. "
+        'Use semicolon to separate multiple targets such as "X86;ARM". '
+        'Defaults to "all" which is to build all supported targets. '
+        "Supported targets: AArch64, AMDGPU, ARM, BPF, Hexagon, Mips, MSP430, NVPTX, PowerPC, "
+        "Sparc, SystemZ, X86, XCore.",
+    )
+    parser.add_argument(
+        "-r", "--run-tests", action="store_true", help="Build and run test cases"
+    )
+    return parser.parse_args()
+def prompt_args():
+    def parse_yn(s: str):
+        table = {"y": True, "n": False}
+        return table.get(s)
+    def parse_int(s: str):
+        try:
+            v = int(s)
+        except ValueError:
+            return None
+        if v <= 0:
+            return None
+    def parse_targets(s: str):
+        if " " in s:
+            return None
+        return s
+    print("No Flags found. Using command line prompts.")
+    print("Alternatively, please call this script with -h for all available options.")
+    auto_build = input_with_check(
+        "Build and install HPVM automatically? [y/n]: ", parse_yn, "Please enter y or n"
+    )
+    if not auto_build:
+        # Just stuff something in the other fields. We won't need them.
+        return Namespace(no_build=True, parallel="", targets="", run_tests=False)
+    threads = input_with_check(
+        "Number of threads: ", parse_int, "Please enter a positive integer"
+    )
+    print(
+        "These build targets are supported: AArch64, AMDGPU, ARM, BPF, Hexagon, "
+        "Mips, MSP430, NVPTX, PowerPC, Sparc, SystemZ, X86, XCore.\n"
+        "If building for multiple targets, seperate options with semicolon:\n"
+        "e.g. X86;ARM"
+    )
+    targets = input_with_check(
+        "Build target: ", parse_targets, "Input shouldn't contain space"
+    )
+    run_tests = input_with_check(
+        "Build and run tests? [y/n]: ", parse_yn, "Please enter y or n"
+    )
+    return Namespace(
+        no_build=not auto_build, parallel=threads, targets=targets, run_tests=run_tests
+    )
+def print_args(args):
+    print("Running with the following options:")
+    print(f"  Automated: {not args.no_build}")
+    print(f"  Threads: {args.parallel}")
+    print(f"  Targets: {args.targets}")
+    print(f"  Run tests: {args.run_tests}")
+def check_download_llvm_clang():
+    if Path("llvm/").is_dir():
+        print("Found LLVM, not extracting it again.")
+    else:
+        if Path(LLVM_TARBALL).is_file():
+            print(f"Found {LLVM_TARBALL}, not downloading it again.")
+        else:
+            print(f"Downloading {LLVM_TARBALL}...")
+            print(f"=============================")
+            check_call([WGET, f"{URL}/{VERSION}/{LLVM_TARBALL}"])
+        check_call(["tar", "xf", LLVM_TARBALL])
+        check_call(["mv", LLVM_DIR, "llvm"])
+    tools = Path("llvm/tools")
+    assert tools.is_dir(), "Problem with LLVM download. Exiting!"
+    if Path(LLVM_TARBALL).is_file():
+        Path(LLVM_TARBALL).unlink()  # Remove tarball
+    environ["LLVM_SRC_ROOT"] = str(ROOT_DIR / "llvm")
+    if (tools / "clang/").is_dir():
+        print("Found clang, not downloading it again.")
+        return
+    chdir(tools)
+    print(f"Downloading {CLANG_TARBALL}...")
+    print(f"=============================")
+    check_call([WGET, f"{URL}/{VERSION}/{CLANG_TARBALL}"])
+    check_call(["tar", "xf", CLANG_TARBALL])
+    check_call(["mv", CLANG_DIR, "clang"])
+    assert Path("clang/").is_dir(), "Problem with clang download. Exiting!"
+    if Path(CLANG_TARBALL).is_file():
+        Path(CLANG_TARBALL).unlink()
+    chdir(ROOT_DIR)
+def link_and_patch():
+    from os import symlink
+    hpvm = ROOT_DIR / "llvm/tools/hpvm"
+    print("Adding HPVM sources to tree...")
+    makedirs(hpvm, exist_ok=True)
+    for link in LINKS:
+        if not (hpvm / link).exists():
+            print(ROOT_DIR / link, hpvm / link)
+            symlink(ROOT_DIR / link, hpvm / link)
+    print("Applying HPVM patches...")
+    chdir("llvm_patches")
+    check_call(["bash", "./construct_patch.sh"])
+    check_call(["bash", "./apply_patch.sh"])
+    print("Patches applied.")
+    chdir("..")
+def maybe_build(build: bool, nthreads: int, targets: str, build_test_targets: bool):
+    if not build:
+        print(
+            """
+HPVM not installed.
+To complete installation, follow these instructions:
+- Create and navigate to a folder "./build" 
+- Run "cmake ../llvm [options]". Find potential options in README.md.
+- Run "make -j<number of threads> approxhpvm.py" and then "make install"
+For more details refer to README.md.
+"""
+        )
+        return
+    print("Now building...")
+    print(f"Using {nthreads} threads to build HPVM.")
+    makedirs(BUILD_DIR, exist_ok=True)
+    chdir(BUILD_DIR)
+    cmake_args = [
+        "cmake",
+        "../llvm",
+        "-DCMAKE_C_COMPILER=gcc",
+        "-DCMAKE_CXX_COMPILER=g++",
+        f"-DLLVM_TARGETS_TO_BUILD={targets}",
+    ]
+    print(f"CMake: {' '.join(cmake_args)}")
+    print(f"=============================")
+    check_call(cmake_args)
+    make_args = ["make", f"-j{nthreads}", *MAKE_TARGETS]
+    if build_test_targets:
+        make_args += MAKE_TEST_TARGETS
+    print(f"Make: {' '.join(make_args)}")
+    print(f"=============================")
+    check_call(make_args)
+    chdir(ROOT_DIR)
+def run_tests():
+    chdir(BUILD_DIR)
+    # Run regression tests
+    check_call([LLVM_LIT, "-v", TEST_DIR / "regressionTests"])
+    # Run unit tests
+    check_call([LLVM_LIT, "-v", TEST_DIR / "unitTests"])
+def input_with_check(prompt: str, parse, prompt_when_invalid: str):
+    input_str = input(prompt)
+    value = parse(input_str)
+    while value is None:
+        print(f"{prompt_when_invalid}; got {input_str}")
+        input_str = input(prompt)
+        value = parse(input_str)
+    return value
+def main():
+    from sys import argv
+    # Don't parse args if no args given -- use prompt mode
+    args = prompt_args() if len(argv) == 1 else parse_args()
+    print_args(args)
+    check_download_llvm_clang()
+    link_and_patch()
+    maybe_build(not args.no_build, args.parallel, args.targets, args.run_tests)
+    if args.run_tests:
+        run_tests()
+    else:
+        print("Skipping tests.")
+if __name__ == "__main__":
+    main()
--- a/hpvm/scripts/llvm_installer.sh
+++ b/hpvm/scripts/llvm_installer.sh
-#!/bin/bash
-function read_yn {
-  read -p "$1 [y/n]: " read_value
-  while [ ! $read_value == "y" ] && [ ! $read_value == "n" ]; do
-    echo "Please answer y or n; got $read_value"
-    read -p "$1 [y/n]:" read_value
-  done
-  eval $2=$read_value
-}
-VERSION="9.0.0"
-URL="http://releases.llvm.org"
-WGET=wget
-CURRENT_DIR=`pwd`
-INSTALL_DIR=`pwd`/install
-BUILD_DIR=$CURRENT_DIR/build
-# Using 2 threads by default
-NUM_THREADS=2
-SUFFIX=".tar.xz"
-CLANG_SRC="cfe-$VERSION.src"
-LLVM_SRC="llvm-$VERSION.src"
-HPVM_RT=hpvm-rt/hpvm-rt.bc
-TARGET=all
-TARGET_INPUT=all
-FLAGGED=false
-# Get flags
-while getopts 'hmj:t:' opt; do
-  case $opt in
-    h) 
-      echo
-      echo
-      echo "This is the help menu for HPVM installation"
-      echo
-      echo "There are 3 options for installation:"
-      echo
-      echo "-m is a manual installation flag. This will require you to install HPVM manually by running cmake and make manually." 
-      echo "For more details, refer to README.md. Defaults to automatic installation."
-      echo
-      echo "-j is the threads flag. Accepts one argument: how many threads to build with." 
-      echo "To build with 2 threads, enter -j2. Defaults to 2 threads."
-      echo
-      echo "-t is the build target flag. Accepts one argument: which build target(s) you would like to build to." 
-      echo "For single target, enter -a ARM. For multiple targets, enter -t \"X86;ARM\"." 
-      echo "Supports the following targets: AArch64, AMDGPU, ARM, BPF, Hexagon, Mips, MSP430, NVPTX, PowerPC, Sparc, SystemZ, X86, XCore."
-      echo "Defaults to targeting all supported architectures."
-      echo
-      echo "If no flags are provided, the script will use command line prompts for all options."
-      echo
-      exit
-      ;;
-    m) 
-      AUTOMATE=false
-      FLAGGED=true
-      ;;
-    j) 
-      if ! [[ $OPTARG =~ ^[0-9]+$ ]]; then
-        echo "Invalid argument for # of threads: $OPTARG"
-        exit -1;
-      else
-        NUM_THREADS=$OPTARG
-        FLAGGED=true
-      fi
-      ;;
-    t) 
-      TARGET=$OPTARG
-      FLAGGED=true
-      ;;
-  esac
-done
-if $FLAGGED; then
-  echo "Running with the following options:"
-  echo Automated: $AUTOMATE
-  echo Threads: $NUM_THREADS
-  echo Targets: $TARGET
-  echo
-else
-  echo "No Flags found. Using command line prompts."
-  read -p "Build and install HPVM automatically? (y or n): " AUTOMATE_INPUT
-  if [[ $AUTOMATE_INPUT == "" ]]; then
-    echo "No input given. Using default: $AUTOMATE"
-  elif [[ ! $AUTOMATE_INPUT == "y" ]] && [[ ! $AUTOMATE_INPUT == "n" ]]; then 
-    echo "Invalid input. Using default: $AUTOMATE"
-  elif [[ $AUTOMATE_INPUT == "n" ]]; then
-    AUTOMATE=false
-  fi
-  echo
-  read -p "Number of threads: " NUM_THREADS_INPUT
-  if [[ $NUM_THREADS_INPUT == "" ]]; then
-    echo "No input given. Using default: $NUM_THREADS"
-  elif ! [[ $NUM_THREADS_INPUT =~ ^[0-9]+$ ]]; then
-    echo "Given input is not an integer. Using default: $NUM_THREADS"
-  elif [ ! $NUM_THREADS_INPUT -gt 0 ]; then
-    echo "Given input is not greater than 0. Using default: $NUM_THREADS"
-  else
-    NUM_THREADS=$NUM_THREADS_INPUT
-  fi
-  echo
-  echo 
-  echo "Supports the following options: AArch64, AMDGPU, ARM, BPF, Hexagon, Mips, MSP430, NVPTX, PowerPC, Sparc, SystemZ, X86, XCore."
-  echo "If building for multiple targets, seperate options with semicolon:"
-  echo "e.g. X86;ARM"
-  read -p "Build target: " TARGET_INPUT
-  if [[ $TARGET_INPUT == "" ]]; then
-    echo "No input given. Using default: $TARGET"
-  else
-    TARGET=$TARGET_INPUT
-  fi
-  echo
-  echo "Running with the following options:"
-  echo Automated: $AUTOMATE
-  echo Threads: $NUM_THREADS
-  echo Targets: $TARGET
-  echo
-fi
-if [ -d $LLVM_SRC ]; then
-    echo Found $LLVM_SRC, not dowloading it again!
-elif [ -d llvm ]; then
-    echo Found LLVM, not downloading it again!
-else
-    echo $WGET $URL/$VERSION/$LLVM_SRC$SUFFIX
-    $WGET $URL/$VERSION/$LLVM_SRC$SUFFIX
-    tar xf $LLVM_SRC$SUFFIX
-    rm $LLVM_SRC$SUFFIX
-fi
-if [ -d $LLVM_SRC ]; then
-    echo Everything looks sane.
-    mv $LLVM_SRC llvm
-elif [ -d llvm ]; then
-    echo Everything looks sane.
-else
-    echo Problem with LLVM download. Exiting!
-    exit
-fi
-LLVM_SRC=llvm
-if [ -d $CURRENT_DIR/$LLVM_SRC/tools ]; then
-    cd $CURRENT_DIR/$LLVM_SRC/tools
-    echo In tools.
-else
-    echo Something is wrong with LLVM checkout. Exiting!
-    exit 1
-fi
-if [ -d clang ]; then
-    echo Found clang! Not downloading clang again.
-else
-    $WGET $URL/$VERSION/$CLANG_SRC$SUFFIX
-    tar xf $CLANG_SRC$SUFFIX
-    rm $CLANG_SRC$SUFFIX
-    mv $CLANG_SRC clang
-    if [ -d clang ]; then
-	echo Everything looks sane.
-    else
-	echo Problem with clang download. Exiting!
-	exit
-    fi
-fi
-cd $CURRENT_DIR
-HPVM_DIR=$CURRENT_DIR/$LLVM_SRC/tools/hpvm
-if [ ! -d $HPVM_DIR ]; then
-  echo Adding HPVM sources to tree
-  mkdir -p $HPVM_DIR
-  ln -s $CURRENT_DIR/CMakeLists.txt $HPVM_DIR
-  ln -s $CURRENT_DIR/cmake $HPVM_DIR/
-  ln -s $CURRENT_DIR/include $HPVM_DIR/
-  ln -s $CURRENT_DIR/lib $HPVM_DIR/
-  ln -s $CURRENT_DIR/projects $HPVM_DIR/
-  ln -s $CURRENT_DIR/test $HPVM_DIR/
-  ln -s $CURRENT_DIR/tools $HPVM_DIR/
-else
-  echo $CURRENT_DIR/$LLVM_SRC/tools/hpvm exists.
-fi
-export LLVM_SRC_ROOT=$CURRENT_DIR/$LLVM_SRC
-echo Applying HPVM patches
-cd $CURRENT_DIR/llvm_patches
-/bin/bash ./construct_patch.sh
-/bin/bash ./apply_patch.sh
-echo Patches applied.
-if ! $AUTOMATE ; then
-  echo
-  echo "HPVM not installed."
-  echo "To complete installation, follow these instructions:"
-  echo "  - Create and navigate to a folder \"./build\" "
-  echo "  - Run \"cmake ../llvm [options]\". Find potential options in README.md."
-  echo "  - Run \"make -j<number of threads> approxhpvm.py\" and then \"make install\""
-  echo "For more details refer to README.md."
-  echo 
-  echo "Exiting."
-  exit  
-fi
-echo
-echo Now building...
-echo Using $NUM_THREADS threads to build HPVM.
-echo
-cd $CURRENT_DIR
-if [ ! -d $BUILD_DIR ]; then
-  mkdir -p $BUILD_DIR
-fi
-if [ ! -d $INSTALL_DIR ]; then
-  mkdir -p $INSTALL_DIR
-fi
-export PATH=$BUILD_DIR/bin:$PATH
-cd $BUILD_DIR
-echo cmake ../$LLVM_SRC -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DLLVM_TARGETS_TO_BUILD=$TARGET  -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR
-cmake ../$LLVM_SRC -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DLLVM_TARGETS_TO_BUILD=$TARGET  -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR
-echo make -j$NUM_THREADS approxhpvm.py
-make -j$NUM_THREADS approxhpvm.py
-#make install
-if [ -f $BUILD_DIR/tools/hpvm/projects/$HPVM_RT ]; then
-    true
-else
-    echo $BUILD_DIR/tools/hpvm/projects/$HPVM_RT
-    echo HPVM not installed properly.
-    exit 0
-fi
-cd $CURRENT_DIR