From ffb352622a570f00868233b731626ecda73e9d6b Mon Sep 17 00:00:00 2001
From: Yifan Zhao <yifanz16@illinois.edu>
Date: Thu, 25 Mar 2021 04:30:15 -0500
Subject: [PATCH] Updated accuracy checker and README
---
hpvm/test/README.md | 39 ++++++++++++++-----
.../test/dnn_benchmarks/hpvm-c/CMakeLists.txt | 13 ++++---
.../dnn_benchmarks/hpvm-c/check_dnn_acc.py | 26 ++++++++++++-
3 files changed, 62 insertions(+), 16 deletions(-)
diff --git a/hpvm/test/README.md b/hpvm/test/README.md
index f7070d6b70..1bfa919c1d 100644
--- a/hpvm/test/README.md
+++ b/hpvm/test/README.md
@@ -14,6 +14,12 @@ This directory is organized as follows:
* `dnn_benchmarks/hpvm-c` contains the HPVM-C version of these DNNs.
Their organization and usage are similar to the benchmarks under `benchmarks/`.
+
+ Each subfolder contains a DNN with 2 versions (2 `.cpp` files):
+ the `tensor`-targeted version which compiles to `tensor_runtime`,
+ and the `cudnn`-targeted version which compiles to operators in `cuDNN`
+ (has `_cudnn` in name).
+
* `dnn_benchmarks/keras` contains these DNNs implemented in Keras,
and code for generating them down to HPVM-C (testing Keras frontend).
* `dnn_benchmarks/pytorch` contains these DNNs in PyTorch
@@ -23,16 +29,33 @@ This directory is organized as follows:
## Running Test Cases and Benchmarks
-The easiest way to run `unitTests/` and `regressionTests/` is
-to build the target `check-hpvm` in the global build directory: `make -j check-hpvm`.
-`check-hpvm` doesn't automatically run `benchmarks/` and `dnn_benchmarks` as they are extremely time-consuming.
+The easiest way to run tests is to use `make` targets,
+which will also take care of all compilation of test cases and test fixtures.
+The following targets runs these tests respectively:
+
+* `make -j check-hpvm-pass` runs tests in `hpvm_pass`: `hpvm_pass/**/*.ll`.
+ These are regression and unit tests for HPVM passes.
+* `make -j check-hpvm-dnn` runs all 20 DNN benchmarks under `dnn_benchmarks/hpvm-c`
+ (10 DNNs x 2 versions) and validates their accuracy.
+
+ *Note* that this is quite time-consuming due to the size of DNNs and datasets.
+ Depending on your hardware capability, this test can take 5-30 minutes.
+ Also, this is set to run sequentially out of GPU memory concerns.
+
+Underneath, `llvm-lit` is used to discover and run the tests.
`benchmarks/` can only be compiled in-source with `make`.
We are working to migrate it into the `cmake` system.
+## Compiling Benchmarks
+
+This section explains how to compile the benchmarks without running them as tests.
+
### HPVM-C DNN Benchmarks
-To build all `dnn_benchmarks/hpvm-c`, use `make -j dnn_benchmarks`.
+To build (not run) all `dnn_benchmarks/hpvm-c`, use `make -j dnn_benchmarks`.
+For each benchmark `${bench_name}`, the binary is generated at
+`${build_dir}/tools/hpvm/test/dnn_benchmarks/hpvm-c/${bench_name}`.
Alternatively, it's possible to build just 1 DNN benchmark.
The output of CMake shows a list of these benchmarks as target names, starting with
@@ -56,10 +79,6 @@ Currently, there are 20 of them. These are:
`_cudnn` suffix indicates the code is generated onto cuDNN functions.
Otherwise they are generated to `tensor_runtime` DNN functions which are hand-written in CUDA.
-### DNN Frontends
-
-TODO: figure out how to
+### TODO: figure out how to
-1. Auto run all hpvm-c DNN benchmarks
-2. Compare the output accuracy to groundtruth
-3. Auto run Keras and PyTorch tests (generating, compiling and running all DNNs)
+1. Auto run Keras and PyTorch tests (generating, compiling and running all DNNs)
diff --git a/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt b/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt
index 640e7d9a5c..a9384956b7 100644
--- a/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt
+++ b/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt
@@ -27,11 +27,14 @@ function(compile_hpvm_c bin_filename src_filepath codegen_target)
endfunction(compile_hpvm_c)
# Install an accuracy comparator under build/bin
-file(
- COPY check_dnn_acc.py
- DESTINATION ${LLVM_BINARY_DIR}/${LLVM_TOOLS_INSTALL_DIR}
- FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
+set(BIN_DIR ${LLVM_BINARY_DIR}/${LLVM_TOOLS_INSTALL_DIR})
+add_custom_command(
+ OUTPUT ${BIN_DIR}/check_dnn_acc.py
+ COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py ${BIN_DIR}
+ COMMAND chmod +x ${BIN_DIR}/check_dnn_acc.py
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py
)
+
function(hpvm_add_dnn_test benchmark_target)
# llvm_test_run* composes a .test file with the RUN line needed by llvm-lit
# No need to give binary path yet;
@@ -62,5 +65,5 @@ foreach(dir ${entries})
endforeach(dir)
message(STATUS "List of HPVM-C DNN benchmarks: ${test_compile_targets}")
-add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets})
+add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets} ${BIN_DIR}/check_dnn_acc.py)
message(STATUS "Target name for compiling all DNN benchmarks: dnn_benchmarks")
diff --git a/hpvm/test/dnn_benchmarks/hpvm-c/check_dnn_acc.py b/hpvm/test/dnn_benchmarks/hpvm-c/check_dnn_acc.py
index ec3d79f368..745836a328 100644
--- a/hpvm/test/dnn_benchmarks/hpvm-c/check_dnn_acc.py
+++ b/hpvm/test/dnn_benchmarks/hpvm-c/check_dnn_acc.py
@@ -1,7 +1,31 @@
#!/usr/bin/env python3
from sys import argv
+network_accuracies = {
+ "alexnet2_cifar10": 84.98,
+ "alexnet_cifar10": 79.28,
+ "alexnet_imagenet": 56.30,
+ "lenet_mnist": 98.70,
+ "mobilenet_cifar10": 84.42,
+ "resnet18_cifar10": 89.56,
+ "resnet50_imagenet": 75.10,
+ "vgg16_cifar10": 89.96,
+ "vgg16_cifar100": 66.50,
+ "vgg16_imagenet": 69.46,
+}
+
+
+def almost_equal(x1, x2):
+ return abs(x1 - x2) < 1e-4
+
+
_, acc_file, network_name = argv
+# cudnn version should have the same accuracy as non-cudnn version.
+network_name = network_name.replace("_cudnn", "")
with open(acc_file) as f:
obtained_acc = float(f.read().strip())
-print(network_name, obtained_acc)
+target_acc = network_accuracies[network_name]
+if not almost_equal(target_acc, obtained_acc):
+ raise ValueError(
+ f"Accuracy mismatch. Obtained: {obtained_acc}, target: {target_acc}"
+ )
--
GitLab