From 87284f7b41b0a9dd91dbd271c1921105b95d15c7 Mon Sep 17 00:00:00 2001
From: Yifan Zhao <yifanz16@illinois.edu>
Date: Wed, 17 Mar 2021 22:08:05 -0500
Subject: [PATCH] Fixed test system and updated README
---
hpvm/test/CMakeLists.txt | 140 ++-----------------
hpvm/test/README.md | 70 +++++++++-
hpvm/test/lit.cfg.py | 288 +--------------------------------------
3 files changed, 82 insertions(+), 416 deletions(-)
diff --git a/hpvm/test/CMakeLists.txt b/hpvm/test/CMakeLists.txt
index 654197c16c..4c96ee124f 100644
--- a/hpvm/test/CMakeLists.txt
+++ b/hpvm/test/CMakeLists.txt
@@ -25,135 +25,23 @@ configure_lit_site_cfg(
# Set the depends list as a variable so that it can grow conditionally.
# NOTE: Sync the substitutions in test/lit.cfg when adding to this list.
-set(LLVM_TEST_DEPENDS
- BugpointPasses
- FileCheck
- LLVMHello
- UnitTests
- bugpoint
- count
- llc
- lli
- lli-child-target
- llvm-addr2line
- llvm-ar
- llvm-as
- llvm-bcanalyzer
- llvm-c-test
- llvm-cat
- llvm-cfi-verify
- llvm-config
- llvm-cov
- llvm-cvtres
- llvm-cxxdump
- llvm-cxxfilt
- llvm-cxxmap
- llvm-diff
- llvm-dis
- llvm-dlltool
- dsymutil
- llvm-dwarfdump
- llvm-dwp
- llvm-elfabi
- llvm-exegesis
- llvm-extract
- llvm-isel-fuzzer
- llvm-jitlink
- llvm-lib
- llvm-link
- llvm-lipo
- llvm-lto2
- llvm-mc
- llvm-mca
- llvm-modextract
- llvm-mt
- llvm-nm
- llvm-objcopy
- llvm-objdump
- llvm-opt-fuzzer
- llvm-opt-report
- llvm-pdbutil
- llvm-profdata
- llvm-ranlib
- llvm-rc
- llvm-readobj
- llvm-readelf
- llvm-rtdyld
- llvm-size
- llvm-split
- llvm-strings
- llvm-strip
- llvm-symbolizer
- llvm-tblgen
- llvm-undname
- llvm-xray
- not
- obj2yaml
- opt
- sancov
- sanstats
- verify-uselistorder
- yaml-bench
- yaml2obj
- )
-
-if(TARGET llvm-lto)
- set(LLVM_TEST_DEPENDS ${LLVM_TEST_DEPENDS} llvm-lto)
-endif()
-
-# If Intel JIT events are supported, depend on a tool that tests the listener.
-if( LLVM_USE_INTEL_JITEVENTS )
- set(LLVM_TEST_DEPENDS ${LLVM_TEST_DEPENDS} llvm-jitlistener)
-endif( LLVM_USE_INTEL_JITEVENTS )
-
-if(TARGET LLVMgold)
- set(LLVM_TEST_DEPENDS ${LLVM_TEST_DEPENDS} LLVMgold)
-endif()
-
-if(TARGET llvm-go)
- set(LLVM_TEST_DEPENDS ${LLVM_TEST_DEPENDS} llvm-go)
-endif()
-
-if(TARGET LTO)
- set(LLVM_TEST_DEPENDS ${LLVM_TEST_DEPENDS} LTO)
-endif()
-
-if(TARGET ocaml_llvm)
- # Clear all non-OCaml cross-target dependencies when building out-of-tree.
- if(LLVM_OCAML_OUT_OF_TREE)
- set(LLVM_TEST_DEPENDS)
- endif()
-
- set(LLVM_TEST_DEPENDS ${LLVM_TEST_DEPENDS}
- ocaml_llvm
- ocaml_llvm_all_backends
- ocaml_llvm_analysis
- ocaml_llvm_bitreader
- ocaml_llvm_bitwriter
- ocaml_llvm_executionengine
- ocaml_llvm_irreader
- ocaml_llvm_linker
- ocaml_llvm_target
- ocaml_llvm_ipo
- ocaml_llvm_passmgr_builder
- ocaml_llvm_scalar_opts
- ocaml_llvm_transform_utils
- ocaml_llvm_vectorize
- )
-endif()
-
-#add_custom_target(llvm-test-depends DEPENDS ${LLVM_TEST_DEPENDS})
-#set_target_properties(llvm-test-depends PROPERTIES FOLDER "Tests")
-
-add_lit_testsuite(check-hpvm "Running the LLVM regression tests"
+set(
+ HPVM_TEST_DEPENDS
+ opt hpvm-rt.bc
+ # Passes:
+ LLVMGenHPVM LLVMBuildDFG LLVMLocalMem LLVMClearDFG
+ LLVMDFG2LLVM_CPU LLVMDFG2LLVM_OpenCL
+ # Test utils:
+ FileCheck count not
+)
+
+add_lit_testsuite(check-hpvm "Running the HPVM regression tests"
${CMAKE_CURRENT_BINARY_DIR}
- DEPENDS ${LLVM_TEST_DEPENDS}
- )
+ DEPENDS ${HPVM_TEST_DEPENDS}
+)
set_target_properties(check-hpvm PROPERTIES FOLDER "Tests")
-add_lit_testsuites(HPVM ${CMAKE_CURRENT_SOURCE_DIR}
- DEPENDS ${LLVM_TEST_DEPENDS}
- )
+add_lit_testsuites(HPVM ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${HPVM_TEST_DEPENDS})
# Setup a legacy alias for 'check-llvm'. This will likely change to be an
# alias for 'check-all' at some point in the future.
diff --git a/hpvm/test/README.md b/hpvm/test/README.md
index 801af4c15a..7e8b408a0c 100644
--- a/hpvm/test/README.md
+++ b/hpvm/test/README.md
@@ -1,5 +1,65 @@
-# Test Directory Organization
-The test directory is organized as follows:
-* unitTests: Includes unit tests for HPVM.
-* regressionTests: Includes regression tests for HPVM.
-* benchmarks: Includes the current benchmarks and a template, as well as directions for compiling and running HPVM benchmarks.
+# HPVM Test and Benchmarks
+
+## Directory Organization
+
+This directory is organized as follows:
+
+* `unitTests/` and `regressionTests/`: unit and regression tests for HPVM.
+ These are LLVM-bitcode test cases for HPVM passes.
+
+* `benchmarks/`: includes a few applications written in HPVM-C, a template, and directions for compiling and running these benchmarks.
+
+* `dnn_benchmarks/`: ten (10) DNN benchmarks in HPVM-C, Keras and PyTorch, supported by ApproxHPVM.
+ This tests HPVM as well as the Keras and PyTorch frontends.
+
+ * `dnn_benchmarks/hpvm-c` contains the HPVM-C version of these DNNs.
+ Their organization and usage are similar to the benchmarks under `benchmarks/`.
+ * `dnn_benchmarks/keras` contains these DNNs implemented in Keras,
+ and code for generating them down to HPVM-C (testing Keras frontend).
+ * `dnn_benchmarks/pytorch` contains these DNNs in PyTorch
+ and code for generating them down to HPVM-C (testing PyTorch/ONNX frontend).
+
+ The code generated from Keras and PyTorch frontend should be largely similar and functionally equivalent.
+
+## Running Test Cases and Benchmarks
+
+The easiest way to run `unitTests/` and `regressionTests/` is
+to build the target `hpvm-check` in the global build directory: `make -j hpvm-check`.
+`hpvm-check` doesn't automatically run `benchmarks/` and `dnn_benchmarks` as they are extremely time-consuming.
+
+`benchmarks/` can only be compiled in-source with `make`.
+We are working to migrate it into the `cmake` system.
+
+### HPVM-C DNN Benchmarks
+
+To build all `dnn_benchmarks/hpvm-c`, use `make -j dnn_benchmarks`.
+
+Alternatively, it's possible to build just 1 DNN benchmark.
+The output of CMake shows a list of these benchmarks as target names, starting with
+> List of test dnn benchmarks: alexnet2_cifar10;alexnet2_cifar10...
+
+Currently, there are 20 of them. These are:
+
+| | |
+|-------------------|-------------------------|
+| lenet_mnist | lenet_mnist_cudnn |
+| alexnet_cifar10 | alexnet_cifar10_cudnn |
+| alexnet2_cifar10 | alexnet2_cifar10_cudnn |
+| vgg16_cifar10 | vgg16_cifar10_cudnn |
+| vgg16_cifar100 | vgg16_cifar100_cudnn |
+| mobilenet_cifar10 | mobilenet_cifar10_cudnn |
+| resnet18_cifar10 | resnet18_cifar10_cudnn |
+| alexnet_imagenet | alexnet_imagenet_cudnn |
+| vgg16_imagenet | vgg16_imagenet_cudnn |
+| resnet50_imagenet | resnet50_imagenet_cudnn |
+
+`_cudnn` suffix indicates the code is generated onto cuDNN functions.
+Otherwise they are generated to `tensor_runtime` DNN functions which are hand-written in CUDA.
+
+### DNN Frontends
+
+TODO: figure out how to
+
+1. Auto run all hpvm-c DNN benchmarks
+2. Compare the output accuracy to groundtruth
+3. Auto run Keras and PyTorch tests (generating, compiling and running all DNNs)
diff --git a/hpvm/test/lit.cfg.py b/hpvm/test/lit.cfg.py
index 1f78a88103..b4e55fbeae 100644
--- a/hpvm/test/lit.cfg.py
+++ b/hpvm/test/lit.cfg.py
@@ -4,15 +4,10 @@
import os
import sys
-import re
-import platform
-import subprocess
import lit.util
import lit.formats
from lit.llvm import llvm_config
-from lit.llvm.subst import FindTool
-from lit.llvm.subst import ToolSubst
# name: The name of this test suite.
config.name = 'HPVM'
@@ -22,12 +17,12 @@ config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
# suffixes: A list of file extensions to treat as test files. This is overriden
# by individual lit.local.cfg files in the test subdirectories.
-config.suffixes = ['.ll', '.c', '.cxx', '.test', '.txt', '.s', '.mir']
+config.suffixes = ['.ll']
# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
# subdirectories contain auxiliary inputs for various tests in their parent
# directories.
-config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']
+config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt', 'benchmarks', 'dnn_benchmarks']
# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)
@@ -42,292 +37,15 @@ llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
llvm_config.with_system_environment(
['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP', 'ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH'])
-
-# Set up OCAMLPATH to include newly built OCaml libraries.
-top_ocaml_lib = os.path.join(config.llvm_lib_dir, 'ocaml')
-llvm_ocaml_lib = os.path.join(top_ocaml_lib, 'llvm')
-
-llvm_config.with_system_environment('OCAMLPATH')
-llvm_config.with_environment('OCAMLPATH', top_ocaml_lib, append_path=True)
-llvm_config.with_environment('OCAMLPATH', llvm_ocaml_lib, append_path=True)
-
-llvm_config.with_system_environment('CAML_LD_LIBRARY_PATH')
-llvm_config.with_environment(
- 'CAML_LD_LIBRARY_PATH', llvm_ocaml_lib, append_path=True)
-
-
-# Set up OCAMLRUNPARAM to enable backtraces in OCaml tests.
-llvm_config.with_environment('OCAMLRUNPARAM', 'b')
-
-# Provide the path to asan runtime lib 'libclang_rt.asan_osx_dynamic.dylib' if
-# available. This is darwin specific since it's currently only needed on darwin.
-
-
-def get_asan_rtlib():
- if not 'Address' in config.llvm_use_sanitizer or \
- not 'Darwin' in config.host_os or \
- not 'x86' in config.host_triple:
- return ''
- try:
- import glob
- except:
- print('glob module not found, skipping get_asan_rtlib() lookup')
- return ''
- # The libclang_rt.asan_osx_dynamic.dylib path is obtained using the relative
- # path from the host cc.
- host_lib_dir = os.path.join(os.path.dirname(config.host_cc), '../lib')
- asan_dylib_dir_pattern = host_lib_dir + \
- '/clang/*/lib/darwin/libclang_rt.asan_osx_dynamic.dylib'
- found_dylibs = glob.glob(asan_dylib_dir_pattern)
- if len(found_dylibs) != 1:
- return ''
- return found_dylibs[0]
-
-
llvm_config.use_default_substitutions()
-# Add site-specific substitutions.
-config.substitutions.append(('%llvmshlibdir', config.llvm_shlib_dir))
-config.substitutions.append(('%shlibext', config.llvm_shlib_ext))
-config.substitutions.append(('%exeext', config.llvm_exe_ext))
-
-
-lli_args = []
-# The target triple used by default by lli is the process target triple (some
-# triple appropriate for generating code for the current process) but because
-# we don't support COFF in MCJIT well enough for the tests, force ELF format on
-# Windows. FIXME: the process target triple should be used here, but this is
-# difficult to obtain on Windows.
-if re.search(r'cygwin|windows-gnu|windows-msvc', config.host_triple):
- lli_args = ['-mtriple=' + config.host_triple + '-elf']
-
-llc_args = []
-
-# Similarly, have a macro to use llc with DWARF even when the host is Windows
-if re.search(r'windows-msvc', config.target_triple):
- llc_args = [' -mtriple=' +
- config.target_triple.replace('-msvc', '-gnu')]
-
-# Provide the path to asan runtime lib if available. On darwin, this lib needs
-# to be loaded via DYLD_INSERT_LIBRARIES before libLTO.dylib in case the files
-# to be linked contain instrumented sanitizer code.
-ld64_cmd = config.ld64_executable
-asan_rtlib = get_asan_rtlib()
-if asan_rtlib:
- ld64_cmd = 'DYLD_INSERT_LIBRARIES={} {}'.format(asan_rtlib, ld64_cmd)
-
-ocamlc_command = '%s ocamlc -cclib -L%s %s' % (
- config.ocamlfind_executable, config.llvm_lib_dir, config.ocaml_flags)
-ocamlopt_command = 'true'
-if config.have_ocamlopt:
- ocamlopt_command = '%s ocamlopt -cclib -L%s -cclib -Wl,-rpath,%s %s' % (
- config.ocamlfind_executable, config.llvm_lib_dir, config.llvm_lib_dir, config.ocaml_flags)
-
opt_viewer_cmd = '%s %s/tools/opt-viewer/opt-viewer.py' % (sys.executable, config.llvm_src_root)
-tools = [
- ToolSubst('%lli', FindTool('lli'), post='.', extra_args=lli_args),
- ToolSubst('%llc_dwarf', FindTool('llc'), extra_args=llc_args),
- ToolSubst('%go', config.go_executable, unresolved='ignore'),
- ToolSubst('%gold', config.gold_executable, unresolved='ignore'),
- ToolSubst('%ld64', ld64_cmd, unresolved='ignore'),
- ToolSubst('%ocamlc', ocamlc_command, unresolved='ignore'),
- ToolSubst('%ocamlopt', ocamlopt_command, unresolved='ignore'),
- ToolSubst('%opt-viewer', opt_viewer_cmd),
- ToolSubst('%llvm-objcopy', FindTool('llvm-objcopy')),
- ToolSubst('%llvm-strip', FindTool('llvm-strip')),
-]
-
-# FIXME: Why do we have both `lli` and `%lli` that do slightly different things?
-tools.extend([
- 'dsymutil', 'lli', 'lli-child-target', 'llvm-ar', 'llvm-as',
- 'llvm-bcanalyzer', 'llvm-config', 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres',
- 'llvm-diff', 'llvm-dis', 'llvm-dwarfdump', 'llvm-exegesis', 'llvm-extract',
- 'llvm-isel-fuzzer', 'llvm-jitlink', 'llvm-opt-fuzzer', 'llvm-lib',
- 'llvm-link', 'llvm-lto', 'llvm-lto2', 'llvm-mc', 'llvm-mca',
- 'llvm-modextract', 'llvm-nm', 'llvm-objcopy', 'llvm-objdump',
- 'llvm-pdbutil', 'llvm-profdata', 'llvm-ranlib', 'llvm-rc', 'llvm-readelf',
- 'llvm-readobj', 'llvm-rtdyld', 'llvm-size', 'llvm-split', 'llvm-strings',
- 'llvm-strip', 'llvm-tblgen', 'llvm-undname', 'llvm-c-test', 'llvm-cxxfilt',
- 'llvm-xray', 'yaml2obj', 'obj2yaml', 'yaml-bench', 'verify-uselistorder',
- 'bugpoint', 'llc', 'llvm-symbolizer', 'opt', 'sancov', 'sanstats'])
-
-# The following tools are optional
-tools.extend([
- ToolSubst('llvm-go', unresolved='ignore'),
- ToolSubst('llvm-mt', unresolved='ignore'),
- ToolSubst('Kaleidoscope-Ch3', unresolved='ignore'),
- ToolSubst('Kaleidoscope-Ch4', unresolved='ignore'),
- ToolSubst('Kaleidoscope-Ch5', unresolved='ignore'),
- ToolSubst('Kaleidoscope-Ch6', unresolved='ignore'),
- ToolSubst('Kaleidoscope-Ch7', unresolved='ignore'),
- ToolSubst('Kaleidoscope-Ch8', unresolved='ignore')])
+tools = ['opt']
llvm_config.add_tool_substitutions(tools, config.llvm_tools_dir)
# Targets
-
config.targets = frozenset(config.targets_to_build.split())
-
for arch in config.targets_to_build.split():
config.available_features.add(arch.lower() + '-registered-target')
-
-# Features
-known_arches = ["x86_64", "mips64", "ppc64", "aarch64"]
-if (config.host_ldflags.find("-m32") < 0
- and any(config.llvm_host_triple.startswith(x) for x in known_arches)):
- config.available_features.add("llvm-64-bits")
-
-config.available_features.add("host-byteorder-" + sys.byteorder + "-endian")
-
-if sys.platform in ['win32']:
- # ExecutionEngine, no weak symbols in COFF.
- config.available_features.add('uses_COFF')
-else:
- # Others/can-execute.txt
- config.available_features.add('can-execute')
-
-# Loadable module
-if config.has_plugins:
- config.available_features.add('plugins')
-
-# Static libraries are not built if BUILD_SHARED_LIBS is ON.
-if not config.build_shared_libs and not config.link_llvm_dylib:
- config.available_features.add('static-libs')
-
-def have_cxx_shared_library():
- readobj_exe = lit.util.which('llvm-readobj', config.llvm_tools_dir)
- if not readobj_exe:
- print('llvm-readobj not found')
- return False
-
- try:
- readobj_cmd = subprocess.Popen(
- [readobj_exe, '-needed-libs', readobj_exe], stdout=subprocess.PIPE)
- except OSError:
- print('could not exec llvm-readobj')
- return False
-
- readobj_out = readobj_cmd.stdout.read().decode('ascii')
- readobj_cmd.wait()
-
- regex = re.compile(r'(libc\+\+|libstdc\+\+|msvcp).*\.(so|dylib|dll)')
- needed_libs = False
- for line in readobj_out.splitlines():
- if 'NeededLibraries [' in line:
- needed_libs = True
- if ']' in line:
- needed_libs = False
- if needed_libs and regex.search(line.lower()):
- return True
- return False
-
-if have_cxx_shared_library():
- config.available_features.add('cxx-shared-library')
-
-if config.libcxx_used:
- config.available_features.add('libcxx-used')
-
-# Direct object generation
-if not 'hexagon' in config.target_triple:
- config.available_features.add('object-emission')
-
-# LLVM can be configured with an empty default triple
-# Some tests are "generic" and require a valid default triple
-if config.target_triple:
- config.available_features.add('default_triple')
-
-import subprocess
-
-
-def have_ld_plugin_support():
- if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'LLVMgold' + config.llvm_shlib_ext)):
- return False
-
- ld_cmd = subprocess.Popen(
- [config.gold_executable, '--help'], stdout=subprocess.PIPE, env={'LANG': 'C'})
- ld_out = ld_cmd.stdout.read().decode()
- ld_cmd.wait()
-
- if not '-plugin' in ld_out:
- return False
-
- # check that the used emulations are supported.
- emu_line = [l for l in ld_out.split('\n') if 'supported emulations' in l]
- if len(emu_line) != 1:
- return False
- emu_line = emu_line[0]
- fields = emu_line.split(':')
- if len(fields) != 3:
- return False
- emulations = fields[2].split()
- if 'elf_x86_64' not in emulations:
- return False
- if 'elf32ppc' in emulations:
- config.available_features.add('ld_emu_elf32ppc')
-
- ld_version = subprocess.Popen(
- [config.gold_executable, '--version'], stdout=subprocess.PIPE, env={'LANG': 'C'})
- if not 'GNU gold' in ld_version.stdout.read().decode():
- return False
- ld_version.wait()
-
- return True
-
-
-if have_ld_plugin_support():
- config.available_features.add('ld_plugin')
-
-
-def have_ld64_plugin_support():
- if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'libLTO' + config.llvm_shlib_ext)):
- return False
-
- if config.ld64_executable == '':
- return False
-
- ld_cmd = subprocess.Popen(
- [config.ld64_executable, '-v'], stderr=subprocess.PIPE)
- ld_out = ld_cmd.stderr.read().decode()
- ld_cmd.wait()
-
- if 'ld64' not in ld_out or 'LTO' not in ld_out:
- return False
-
- return True
-
-
-if have_ld64_plugin_support():
- config.available_features.add('ld64_plugin')
-
-# Ask llvm-config about asserts and global-isel.
-llvm_config.feature_config(
- [('--assertion-mode', {'ON': 'asserts'}),
- ('--has-global-isel', {'ON': 'global-isel'})])
-
-if 'darwin' == sys.platform:
- try:
- sysctl_cmd = subprocess.Popen(['sysctl', 'hw.optional.fma'],
- stdout=subprocess.PIPE)
- except OSError:
- print('Could not exec sysctl')
- result = sysctl_cmd.stdout.read().decode('ascii')
- if -1 != result.find('hw.optional.fma: 1'):
- config.available_features.add('fma3')
- sysctl_cmd.wait()
-
-# .debug_frame is not emitted for targeting Windows x64.
-if not re.match(r'^x86_64.*-(windows-gnu|windows-msvc)', config.target_triple):
- config.available_features.add('debug_frame')
-
-if config.have_libxar:
- config.available_features.add('xar')
-
-if config.enable_threads:
- config.available_features.add('thread_support')
-
-if config.llvm_libxml2_enabled:
- config.available_features.add('libxml2')
-
-if config.have_opt_viewer_modules:
- config.available_features.add('have_opt_viewer_modules')
--
GitLab