Updated accuracy checker and README

hpvm/test/README.md

@@ -14,6 +14,12 @@ This directory is organized as follows:
 
   * `dnn_benchmarks/hpvm-c` contains the HPVM-C version of these DNNs.
     Their organization and usage are similar to the benchmarks under `benchmarks/`.
+
+    Each subfolder contains a DNN with 2 versions (2 `.cpp` files):
+    the `tensor`-targeted version which compiles to `tensor_runtime`,
+    and the `cudnn`-targeted version which compiles to operators in `cuDNN`
+    (has `_cudnn` in name).
+
   * `dnn_benchmarks/keras` contains these DNNs implemented in Keras,
     and code for generating them down to HPVM-C (testing Keras frontend).
   * `dnn_benchmarks/pytorch` contains these DNNs in PyTorch
@@ -23,16 +29,33 @@ This directory is organized as follows:
 
 ## Running Test Cases and Benchmarks
 
-The easiest way to run `unitTests/` and `regressionTests/` is
-to build the target `check-hpvm` in the global build directory: `make -j check-hpvm`.
-`check-hpvm` doesn't automatically run `benchmarks/` and `dnn_benchmarks` as they are extremely time-consuming.
+The easiest way to run tests is to use `make` targets,
+which will also take care of all compilation of test cases and test fixtures.
+The following targets runs these tests respectively:
+
+* `make -j check-hpvm-pass` runs tests in `hpvm_pass`: `hpvm_pass/**/*.ll`.
+  These are regression and unit tests for HPVM passes.
+* `make -j check-hpvm-dnn` runs all 20 DNN benchmarks under `dnn_benchmarks/hpvm-c`
+  (10 DNNs x 2 versions) and validates their accuracy.
+
+  *Note* that this is quite time-consuming due to the size of DNNs and datasets.
+  Depending on your hardware capability, this test can take 5-30 minutes.
+  Also, this is set to run sequentially out of GPU memory concerns.
+
+Underneath, `llvm-lit` is used to discover and run the tests.
 
 `benchmarks/` can only be compiled in-source with `make`.
 We are working to migrate it into the `cmake` system.
 
+## Compiling Benchmarks
+
+This section explains how to compile the benchmarks without running them as tests.
+
 ### HPVM-C DNN Benchmarks
 
-To build all `dnn_benchmarks/hpvm-c`, use `make -j dnn_benchmarks`.
+To build (not run) all `dnn_benchmarks/hpvm-c`, use `make -j dnn_benchmarks`.
+For each benchmark `${bench_name}`, the binary is generated at
+`${build_dir}/tools/hpvm/test/dnn_benchmarks/hpvm-c/${bench_name}`.
 
 Alternatively, it's possible to build just 1 DNN benchmark.
 The output of CMake shows a list of these benchmarks as target names, starting with
@@ -56,10 +79,6 @@ Currently, there are 20 of them. These are:
 `_cudnn` suffix indicates the code is generated onto cuDNN functions.
 Otherwise they are generated to `tensor_runtime` DNN functions which are hand-written in CUDA.
 
-### DNN Frontends
-
-TODO: figure out how to
+### TODO: figure out how to
 
-1. Auto run all hpvm-c DNN benchmarks
-2. Compare the output accuracy to groundtruth
-3. Auto run Keras and PyTorch tests (generating, compiling and running all DNNs)
+1. Auto run Keras and PyTorch tests (generating, compiling and running all DNNs)

hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt

@@ -27,11 +27,14 @@ function(compile_hpvm_c bin_filename src_filepath codegen_target)
 endfunction(compile_hpvm_c)
 
 # Install an accuracy comparator under build/bin
-file(
-  COPY check_dnn_acc.py
-  DESTINATION ${LLVM_BINARY_DIR}/${LLVM_TOOLS_INSTALL_DIR}
-  FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
+set(BIN_DIR ${LLVM_BINARY_DIR}/${LLVM_TOOLS_INSTALL_DIR})
+add_custom_command(
+  OUTPUT ${BIN_DIR}/check_dnn_acc.py
+  COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py ${BIN_DIR}
+  COMMAND chmod +x ${BIN_DIR}/check_dnn_acc.py
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py
 )
+
 function(hpvm_add_dnn_test benchmark_target)
   # llvm_test_run* composes a .test file with the RUN line needed by llvm-lit
   # No need to give binary path yet;
@@ -62,5 +65,5 @@ foreach(dir ${entries})
 endforeach(dir)
 
 message(STATUS "List of HPVM-C DNN benchmarks: ${test_compile_targets}")
-add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets})
+add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets} ${BIN_DIR}/check_dnn_acc.py)
 message(STATUS "Target name for compiling all DNN benchmarks: dnn_benchmarks")

hpvm/test/dnn_benchmarks/hpvm-c/check_dnn_acc.py

 #!/usr/bin/env python3
 from sys import argv
 
+network_accuracies = {
+    "alexnet2_cifar10": 84.98,
+    "alexnet_cifar10": 79.28,
+    "alexnet_imagenet": 56.30,
+    "lenet_mnist": 98.70,
+    "mobilenet_cifar10": 84.42,
+    "resnet18_cifar10": 89.56,
+    "resnet50_imagenet": 75.10,
+    "vgg16_cifar10": 89.96,
+    "vgg16_cifar100": 66.50,
+    "vgg16_imagenet": 69.46,
+}
+
+
+def almost_equal(x1, x2):
+    return abs(x1 - x2) < 1e-4
+
+
 _, acc_file, network_name = argv
+# cudnn version should have the same accuracy as non-cudnn version.
+network_name = network_name.replace("_cudnn", "")
 with open(acc_file) as f:
     obtained_acc = float(f.read().strip())
-print(network_name, obtained_acc)
+target_acc = network_accuracies[network_name]
+if not almost_equal(target_acc, obtained_acc):
+    raise ValueError(
+        f"Accuracy mismatch. Obtained: {obtained_acc}, target: {target_acc}"
+    )