Skip to content
Snippets Groups Projects
Commit 0798e39e authored by Yifan Zhao's avatar Yifan Zhao
Browse files

Added automated DNN accuracy test to llvm-lit

parent 2b0f0226
No related branches found
No related tags found
No related merge requests found
# This file is taken from llvm-testsuite:
# https://github.com/llvm/llvm-test-suite/blob/main/cmake/modules/TestFile.cmake
##===- TestSuite.cmake ----------------------------------------------------===##
#
# Defines helper functions to create .test files that describe how to run a
# benchmark and optionally how to prepare before running, how to verify the
# results and how to extract metrics from the output.
#
##===----------------------------------------------------------------------===##
# Specify a "RUN: " line to be put in a .test file. See also llvm_add_test().
macro(llvm_test_run)
CMAKE_PARSE_ARGUMENTS(ARGS "" "RUN_TYPE;EXECUTABLE;WORKDIR" "" ${ARGN})
# If no executable is specified use $EXECUTABLE$ placeholder which will be
# replaced later.
if(NOT DEFINED ARGS_EXECUTABLE)
set(ARGS_EXECUTABLE "$EXECUTABLE$")
endif()
if(NOT DEFINED TESTSCRIPT)
set(TESTSCRIPT "" PARENT_SCOPE)
endif()
if(DEFINED ARGS_WORKDIR)
set(ARGS_EXECUTABLE "cd ${ARGS_WORKDIR} ; ${ARGS_EXECUTABLE}")
endif()
# ARGS_UNPARSED_ARGUMENTS is a semicolon-separated list. Change it into a
# whitespace-separated string.
string(REPLACE ";" " " JOINED_ARGUMENTS "${ARGS_UNPARSED_ARGUMENTS}")
if(NOT DEFINED ARGS_RUN_TYPE OR "${ARGS_RUN_TYPE}" STREQUAL "${TEST_SUITE_RUN_TYPE}")
set(TESTSCRIPT "${TESTSCRIPT}RUN: ${ARGS_EXECUTABLE} ${JOINED_ARGUMENTS}\n")
endif()
endmacro()
# Specify a "VERIFY: " line to be put in a .test file. See also llvm_add_test().
macro(llvm_test_verify)
CMAKE_PARSE_ARGUMENTS(ARGS "" "RUN_TYPE;WORKDIR" "" ${ARGN})
if(NOT DEFINED TESTSCRIPT)
set(TESTSCRIPT "" PARENT_SCOPE)
endif()
# ARGS_UNPARSED_ARGUMENTS is a semicolon-separated list. Change it into a
# whitespace-separated string.
string(REPLACE ";" " " JOINED_ARGUMENTS "${ARGS_UNPARSED_ARGUMENTS}")
if(DEFINED ARGS_WORKDIR)
set(JOINED_ARGUMENTS "cd ${ARGS_WORKDIR} ; ${JOINED_ARGUMENTS}")
endif()
if(NOT DEFINED ARGS_RUN_TYPE OR "${ARGS_RUN_TYPE}" STREQUAL "${TEST_SUITE_RUN_TYPE}")
set(TESTSCRIPT "${TESTSCRIPT}VERIFY: ${JOINED_ARGUMENTS}\n")
endif()
endmacro()
# Specify a "PREPARE: " line to be put in a .test file. See also
# llvm_add_test().
macro(llvm_test_prepare)
CMAKE_PARSE_ARGUMENTS(ARGS "" "RUN_TYPE;WORKDIR" "" ${ARGN})
if(NOT DEFINED TESTSCRIPT)
set(TESTSCRIPT "" PARENT_SCOPE)
endif()
# ARGS_UNPARSED_ARGUMENTS is a semicolon-separated list. Change it into a
# whitespace-separated string.
string(REPLACE ";" " " JOINED_ARGUMENTS "${ARGS_UNPARSED_ARGUMENTS}")
if(DEFINED ARGS_WORKDIR)
set(JOINED_ARGUMENTS "cd ${ARGS_WORKDIR} ; ${JOINED_ARGUMENTS}")
endif()
if(NOT DEFINED ARGS_RUN_TYPE OR "${ARGS_RUN_TYPE}" STREQUAL "${TEST_SUITE_RUN_TYPE}")
set(TESTSCRIPT "${TESTSCRIPT}PREPARE: ${JOINED_ARGUMENTS}\n")
endif()
endmacro()
# Specify a "METRIC: " line to be put in a .test file. See also llvm_add_test().
macro(llvm_test_metric)
CMAKE_PARSE_ARGUMENTS(ARGS "" "RUN_TYPE;METRIC" "" ${ARGN})
if(NOT DEFINED TESTSCRIPT)
set(TESTSCRIPT "" PARENT_SCOPE)
endif()
# ARGS_UNPARSED_ARGUMENTS is a semicolon-separated list. Change it into a
# whitespace-separated string.
string(REPLACE ";" " " JOINED_ARGUMENTS "${ARGS_UNPARSED_ARGUMENTS}")
if(NOT DEFINED ARGS_RUN_TYPE OR "${ARGS_RUN_TYPE}" STREQUAL "${TEST_SUITE_RUN_TYPE}")
set(TESTSCRIPT "${TESTSCRIPT}METRIC: ${ARGS_METRIC}: ${JOINED_ARGUMENTS}\n")
endif()
endmacro()
# Create a .test driver file suitable for LIT.
# The file contents come from previous llvm_test_XXX() calls.
function(llvm_add_test testfile executable)
# Replace $EXECUTABLE$ placeholder.
string(REPLACE "$EXECUTABLE$" "${executable}" TESTSCRIPT "${TESTSCRIPT}")
# Produce .test file
file(GENERATE OUTPUT ${testfile} CONTENT "${TESTSCRIPT}")
# flush the test script
set(TESTSCRIPT "" PARENT_SCOPE)
endfunction()
function(llvm_add_test_for_target target)
llvm_add_test($<TARGET_FILE:${target}>.test $<TARGET_FILE:${target}>)
set(TESTSCRIPT "" PARENT_SCOPE)
endfunction()
\ No newline at end of file
llvm_canonicalize_cmake_booleans(
BUILD_SHARED_LIBS
HAVE_LIBXAR
HAVE_LIBZ
HAVE_OCAMLOPT
HAVE_OCAML_OUNIT
LLVM_ENABLE_DIA_SDK
LLVM_ENABLE_FFI
LLVM_ENABLE_THREADS
LLVM_INCLUDE_GO_TESTS
LLVM_LIBXML2_ENABLED
LLVM_LINK_LLVM_DYLIB
LLVM_TOOL_LTO_BUILD
LLVM_USE_INTEL_JITEVENTS
LLVM_BUILD_EXAMPLES
LLVM_ENABLE_PLUGINS
)
configure_lit_site_cfg(
${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
)
# Set the depends list as a variable so that it can grow conditionally.
# NOTE: Sync the substitutions in test/lit.cfg when adding to this list.
set(
HPVM_TEST_DEPENDS
opt hpvm-rt.bc
# Passes:
LLVMGenHPVM LLVMBuildDFG LLVMLocalMem LLVMClearDFG
LLVMDFG2LLVM_CPU LLVMDFG2LLVM_OpenCL
# Test utils:
FileCheck count not
)
add_lit_testsuite(check-hpvm "Running the HPVM regression tests"
${CMAKE_CURRENT_BINARY_DIR}
DEPENDS ${HPVM_TEST_DEPENDS}
)
set_target_properties(check-hpvm PROPERTIES FOLDER "Tests")
add_lit_testsuites(HPVM ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${HPVM_TEST_DEPENDS})
# HPVM-C benchmarks uses ctest instead of LLVM's test mechanism
# because they are compiled in a custom way
include(CTest)
add_subdirectory(dnn_benchmarks/hpvm-c)
include(../cmake/TestFile.cmake) # Generation of `.test` files in CMake
add_subdirectory(hpvm_pass)
add_subdirectory(dnn_benchmarks)
# --[ llvm-lit test setup
# lit.cfg.py looks for tests in CMAKE_CURRENT_BINARY_DIR (see lit.cfg.py)
# as most of the tests require some kind of compilation / generation
# which is best done over there.
configure_lit_site_cfg(
${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
)
add_subdirectory(hpvm-c)
set(HPVM_TEST_DEPENDS dnn_benchmarks) # Compile all dnn benchmarks to run them
add_lit_testsuite(check-hpvm-dnn "Running HPVM DNNs"
${CMAKE_CURRENT_BINARY_DIR}
DEPENDS ${HPVM_TEST_DEPENDS}
ARGS "-j1" # Run DNN benchmarks sequentially
)
......@@ -14,7 +14,7 @@ configure_file(
set(CONFIG_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include")
set(test_compile_targets "")
function(approxhpvm_py_codegen bin_filename src_filepath codegen_target)
function(compile_hpvm_c bin_filename src_filepath codegen_target)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${bin_filename}
DEPENDS ${src_filepath} approxhpvm.py
......@@ -23,34 +23,42 @@ function(approxhpvm_py_codegen bin_filename src_filepath codegen_target)
-t ${codegen_target} -I ${CONFIG_INCLUDE_DIR} ${ARGV}
)
add_custom_target(${bin_filename} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${bin_filename})
add_test(NAME test_${bin_filename} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${bin_filename})
set_tests_properties(test_${bin_filename} PROPERTIES RUN_SERIAL TRUE)
set(test_compile_targets ${test_compile_targets} ${bin_filename} PARENT_SCOPE)
endfunction(approxhpvm_py_codegen)
endfunction(compile_hpvm_c)
set(test_run_targets "")
function(run_single_benchmark run_target benchmark)
add_custom_target(
${run_target} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${benchmark}
)
add_dependencies(${run_target} ${benchmark})
set(test_run_targets ${test_run_targets} ${run_target} PARENT_SCOPE)
endfunction(run_single_benchmark)
# Install an accuracy comparator under build/bin
file(
COPY check_dnn_acc.py
DESTINATION ${LLVM_BINARY_DIR}/${LLVM_TOOLS_INSTALL_DIR}
FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
)
function(hpvm_add_dnn_test benchmark_target)
# llvm_test_run* composes a .test file with the RUN line needed by llvm-lit
# No need to give binary path yet;
# llvm_add_test_for_target knows the binary path from target name
# Runs the binary with no argument
llvm_test_run()
# Requests comparison of accuracy
llvm_test_run(EXECUTABLE check_dnn_acc.py final_accuracy ${benchmark_target})
# Removes the final_accuracy file
llvm_test_run(EXECUTABLE rm final_accuracy)
# llvm_add_test creates .test file to given output path for given binary.
llvm_add_test(${benchmark_target}.test ${CMAKE_CURRENT_BINARY_DIR}/${benchmark_target})
# TODO: add_dependencies
endfunction(hpvm_add_dnn_test)
file(GLOB entries ./benchmarks/*)
foreach(dir ${entries})
get_filename_component(dirname "${dir}" NAME)
# Generate "tensor"-targeted code
approxhpvm_py_codegen(
compile_hpvm_c(
${dirname} ${dir}/${dirname}.cpp tensor
--conf-file ${dir}/data/tuner_confs.txt
)
# Run tensor binary
run_single_benchmark(run_${dirname} ${dirname})
hpvm_add_dnn_test(${dirname})
# Generate "cudnn"-targeted code
approxhpvm_py_codegen(${dirname}_cudnn ${dir}/${dirname}_cudnn.cpp cudnn)
# Run cudnn binary
run_single_benchmark(run_${dirname}_cudnn ${dirname}_cudnn)
compile_hpvm_c(${dirname}_cudnn ${dir}/${dirname}_cudnn.cpp cudnn)
hpvm_add_dnn_test(${dirname}_cudnn)
endforeach(dir)
message(STATUS "List of HPVM-C DNN benchmarks: ${test_compile_targets}")
......
#!/usr/bin/env python3
from sys import argv
_, acc_file, network_name = argv
with open(acc_file) as f:
obtained_acc = float(f.read().strip())
print(network_name, obtained_acc)
# -*- Python -*-
# Configuration file for the 'lit' test runner.
import os
import lit.formats
from lit.llvm import llvm_config
# name: The name of this test suite.
config.name = "HPVM-DNN"
# testFormat: The test format to use to interpret tests.
config.test_format = lit.formats.ShTest(False)
# suffixes: A list of file extensions to treat as test files. This is overriden
# by individual lit.local.cfg files in the test subdirectories.
config.suffixes = [".test"]
# test_source_root: The root path where tests are located.
# test_exec_root: The root path where tests should be run.
current_source_dir = os.path.dirname(os.path.relpath(__file__, config.llvm_src_root))
current_binary_dir = os.path.join(config.llvm_obj_root, current_source_dir)
config.test_source_root = config.test_exec_root = current_binary_dir
# Tweak the PATH to include the tools dir.
llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
config.llvm_src_root = "@LLVM_SOURCE_DIR@"
config.llvm_obj_root = "@LLVM_BINARY_DIR@"
config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@"
import lit.llvm
lit.llvm.initialize(lit_config, config)
# Let the main config do the real work.
lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")
configure_lit_site_cfg(
${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
)
set(
HPVM_TEST_DEPENDS
opt hpvm-rt.bc
# Passes:
LLVMGenHPVM LLVMBuildDFG LLVMLocalMem LLVMClearDFG
LLVMDFG2LLVM_CPU LLVMDFG2LLVM_OpenCL
# Test utils:
FileCheck count not
)
add_lit_testsuite(check-hpvm-pass "Running HPVM test cases for the passes"
${CMAKE_CURRENT_BINARY_DIR}
DEPENDS ${HPVM_TEST_DEPENDS}
)
......@@ -3,49 +3,32 @@
# Configuration file for the 'lit' test runner.
import os
import sys
import lit.util
import lit.formats
from lit.llvm import llvm_config
# name: The name of this test suite.
config.name = 'HPVM'
config.name = "HPVM-PASS"
# testFormat: The test format to use to interpret tests.
config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
# suffixes: A list of file extensions to treat as test files. This is overriden
# by individual lit.local.cfg files in the test subdirectories.
config.suffixes = ['.ll']
# excludes: A list of directories to exclude from the testsuite. The 'Inputs'
# subdirectories contain auxiliary inputs for various tests in their parent
# directories.
config.excludes = ['Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt', 'benchmarks', 'dnn_benchmarks']
config.suffixes = [".ll"]
# test_source_root: The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)
# test_exec_root: The root path where tests should be run.
config.test_exec_root = os.path.join(config.llvm_obj_root, 'test')
current_source_dir = os.path.dirname(os.path.relpath(__file__, config.llvm_src_root))
current_binary_dir = os.path.join(config.llvm_obj_root, current_source_dir)
config.test_exec_root = current_binary_dir
# Tweak the PATH to include the tools dir.
llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
# Propagate some variables from the host environment.
llvm_config.with_system_environment(
['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP', 'ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH'])
llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
llvm_config.use_default_substitutions()
opt_viewer_cmd = '%s %s/tools/opt-viewer/opt-viewer.py' % (sys.executable, config.llvm_src_root)
tools = ['opt']
tools = ["opt"]
llvm_config.add_tool_substitutions(tools, config.llvm_tools_dir)
# Targets
config.targets = frozenset(config.targets_to_build.split())
for arch in config.targets_to_build.split():
config.available_features.add(arch.lower() + '-registered-target')
from lit.llvm import llvm_config
config.llvm_src_root = "@LLVM_SOURCE_DIR@"
config.llvm_obj_root = "@LLVM_BINARY_DIR@"
config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@"
import lit.llvm
lit.llvm.initialize(lit_config, config)
# Let the main config do the real work.
lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")
@LIT_SITE_CFG_IN_HEADER@
import sys
config.host_triple = "@LLVM_HOST_TRIPLE@"
config.target_triple = "@TARGET_TRIPLE@"
config.llvm_src_root = "@LLVM_SOURCE_DIR@"
config.llvm_obj_root = "@LLVM_BINARY_DIR@"
config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@"
config.llvm_shlib_dir = "@SHLIBDIR@"
config.llvm_shlib_ext = "@SHLIBEXT@"
config.llvm_exe_ext = "@EXEEXT@"
config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
config.python_executable = "@PYTHON_EXECUTABLE@"
config.gold_executable = "@GOLD_EXECUTABLE@"
config.ld64_executable = "@LD64_EXECUTABLE@"
config.ocamlfind_executable = "@OCAMLFIND@"
config.have_ocamlopt = @HAVE_OCAMLOPT@
config.have_ocaml_ounit = @HAVE_OCAML_OUNIT@
config.ocaml_flags = "@OCAMLFLAGS@"
config.include_go_tests = @LLVM_INCLUDE_GO_TESTS@
config.go_executable = "@GO_EXECUTABLE@"
config.enable_shared = @ENABLE_SHARED@
config.enable_assertions = @ENABLE_ASSERTIONS@
config.targets_to_build = "@TARGETS_TO_BUILD@"
config.native_target = "@LLVM_NATIVE_ARCH@"
config.llvm_bindings = "@LLVM_BINDINGS@".split(' ')
config.host_os = "@HOST_OS@"
config.host_cc = "@HOST_CC@"
config.host_cxx = "@HOST_CXX@"
config.host_ldflags = "@HOST_LDFLAGS@"
config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
config.have_zlib = @HAVE_LIBZ@
config.have_libxar = @HAVE_LIBXAR@
config.have_dia_sdk = @LLVM_ENABLE_DIA_SDK@
config.enable_ffi = @LLVM_ENABLE_FFI@
config.build_examples = @LLVM_BUILD_EXAMPLES@
config.enable_threads = @LLVM_ENABLE_THREADS@
config.build_shared_libs = @BUILD_SHARED_LIBS@
config.link_llvm_dylib = @LLVM_LINK_LLVM_DYLIB@
config.llvm_libxml2_enabled = @LLVM_LIBXML2_ENABLED@
config.llvm_host_triple = '@LLVM_HOST_TRIPLE@'
config.host_arch = "@HOST_ARCH@"
config.have_opt_viewer_modules = @LLVM_HAVE_OPT_VIEWER_MODULES@
config.libcxx_used = @LLVM_LIBCXX_USED@
config.has_plugins = @LLVM_ENABLE_PLUGINS@
# Support substitution of the tools_dir with user parameters. This is
# used when we can't determine the tool dir at configuration time.
try:
config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
config.llvm_shlib_dir = config.llvm_shlib_dir % lit_config.params
except KeyError:
e = sys.exc_info()[1]
key, = e.args
lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
import lit.llvm
lit.llvm.initialize(lit_config, config)
# Let the main config do the real work.
lit_config.load_config(config, "@LLVM_SOURCE_DIR@/tools/hpvm/test/lit.cfg.py")
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment