From 13c1799fc36a3911d01b01c2afe9183262ec10c9 Mon Sep 17 00:00:00 2001
From: Yifan Zhao <yifanz16@illinois.edu>
Date: Sat, 3 Apr 2021 13:19:06 -0500
Subject: [PATCH] Added accuracy checking for frontend-generated code
---
.../test/dnn_benchmarks/hpvm-c/CMakeLists.txt | 7 ++++---
.../dnn_benchmarks/pytorch/CMakeLists.txt | 4 ++++
.../pytorch/alexnet2_cifar10.test | 1 +
.../pytorch/alexnet_cifar10.test | 1 +
.../pytorch/alexnet_imagenet.test | 1 +
.../dnn_benchmarks/pytorch/lenet_mnist.test | 1 +
hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py | 4 ++++
.../pytorch/mobilenet_cifar10.test | 1 +
.../pytorch/resnet18_cifar10.test | 1 +
.../pytorch/resnet50_imagenet.test | 1 +
.../dnn_benchmarks/pytorch/test_frontend.py | 19 ++++++++++---------
.../dnn_benchmarks/pytorch/vgg16_cifar10.test | 1 +
.../pytorch/vgg16_cifar100.test | 1 +
.../pytorch/vgg16_imagenet.test | 1 +
14 files changed, 32 insertions(+), 12 deletions(-)
diff --git a/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt b/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt
index 487adc4fa5..f56312e9c3 100644
--- a/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt
+++ b/hpvm/test/dnn_benchmarks/hpvm-c/CMakeLists.txt
@@ -49,16 +49,17 @@ foreach(dir ${entries})
endforeach(dir)
# Install an accuracy comparator under build/bin for test suite.
-set(BIN_DIR ${LLVM_BINARY_DIR}/${LLVM_TOOLS_INSTALL_DIR})
+set(BIN_DIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
add_custom_command(
OUTPUT ${BIN_DIR}/check_dnn_acc.py
COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py ${BIN_DIR}
COMMAND chmod +x ${BIN_DIR}/check_dnn_acc.py
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/check_dnn_acc.py
)
+add_custom_target(check_dnn_acc DEPENDS ${BIN_DIR}/check_dnn_acc.py)
message(STATUS "List of HPVM-C DNN benchmarks: ${test_compile_targets}")
-add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets} ${BIN_DIR}/check_dnn_acc.py)
+add_custom_target(dnn_benchmarks DEPENDS ${test_compile_targets})
message(STATUS "Target name for compiling all DNN benchmarks: dnn_benchmarks")
# --[ llvm-lit test setup
@@ -73,6 +74,6 @@ configure_lit_site_cfg(
)
add_lit_testsuite(check-hpvm-dnn "Running HPVM DNNs"
${CMAKE_CURRENT_BINARY_DIR}
- DEPENDS dnn_benchmarks # Compile all dnn benchmarks to run them
+ DEPENDS dnn_benchmarks check_dnn_acc # Compile all dnn benchmarks to run them
ARGS "-j1" # Run DNN benchmarks sequentially
)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt
index b184d2995b..778593a57d 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt
+++ b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt
@@ -10,5 +10,9 @@ configure_lit_site_cfg(
)
add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for package torch2hpvm"
${CMAKE_CURRENT_BINARY_DIR}
+ # We depend on check_dnn_acc.py defined in ../hpvm-c/
+ # to compare the inference accuracy of our frontend-generated binary
+ # to that of the baseline.
+ DEPENDS check_dnn_acc
ARGS "-j1" # Run frontend generation sequentially
)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test
index 378f8f80eb..4adf30226b 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet2_cifar10.test
@@ -1 +1,2 @@
RUN: test_frontend.py alexnet2_cifar10
+RUN: check_dnn_acc.py final_accuracy alexnet2_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test
index de7283c9a1..cffec91e41 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet_cifar10.test
@@ -1 +1,2 @@
RUN: test_frontend.py alexnet_cifar10
+RUN: check_dnn_acc.py final_accuracy alexnet_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test
index 565f9d6f49..126de1bfe8 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/alexnet_imagenet.test
@@ -1 +1,2 @@
RUN: test_frontend.py alexnet_imagenet
+RUN: check_dnn_acc.py final_accuracy alexnet_imagenet
diff --git a/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test b/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test
index 9c0d3ebbbd..b87a976bcd 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/lenet_mnist.test
@@ -1 +1,2 @@
RUN: test_frontend.py lenet_mnist
+RUN: check_dnn_acc.py final_accuracy lenet_mnist
diff --git a/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py b/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py
index 6a098a5d76..34473d24be 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py
+++ b/hpvm/test/dnn_benchmarks/pytorch/lit.cfg.py
@@ -28,5 +28,9 @@ config.test_exec_root = current_binary_dir
# Tweak the PATH to include the tools dir.
llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
+# Add substitution for check_dnn_acc.py which goes under build/bin.
+llvm_config.add_tool_substitutions(
+ ["check_dnn_acc.py"], os.path.join(config.llvm_obj_root, "bin")
+)
# Add substitution for our main script in this directory.
llvm_config.add_tool_substitutions(["test_frontend.py"], config.test_source_root)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test
index af9065e184..9964887b42 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/mobilenet_cifar10.test
@@ -1 +1,2 @@
RUN: test_frontend.py mobilenet_cifar10
+RUN: check_dnn_acc.py final_accuracy mobilenet_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test
index 5d08067c6e..71e0881a3f 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/resnet18_cifar10.test
@@ -1 +1,2 @@
RUN: test_frontend.py resnet18_cifar10
+RUN: check_dnn_acc.py final_accuracy resnet18_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test
index de08c21c43..b1ff2e6a92 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/resnet50_imagenet.test
@@ -1 +1,2 @@
RUN: test_frontend.py resnet50_imagenet
+RUN: check_dnn_acc.py final_accuracy resnet50_imagenet
diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py b/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py
index 89a49d511d..3c20c6ea5a 100755
--- a/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_frontend.py
@@ -14,16 +14,16 @@ site.addsitedir(os.path.dirname(__file__))
import dnn
benchmarks = {
- "lenet_mnist": (dnn.LeNet, 1, 28, 5000),
- "alexnet_cifar10": (dnn.AlexNet, 3, 32, 5000),
- "alexnet2_cifar10": (dnn.AlexNet2, 3, 32, 5000),
+ "lenet_mnist": (dnn.LeNet, 1, 28, 1000),
+ "alexnet_cifar10": (dnn.AlexNet, 3, 32, 500),
+ "alexnet2_cifar10": (dnn.AlexNet2, 3, 32, 500),
"alexnet_imagenet": (dnn.AlexNetImageNet, 3, 224, 500),
- "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 5000),
- "resnet18_cifar10": (dnn.ResNet18, 3, 32, 5000),
- "resnet50_imagenet": (dnn.ResNet50, 3, 224, 100),
- "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 5000),
- "vgg16_cifar100": (dnn.VGG16Cifar100, 3, 32, 5000),
- "vgg16_imagenet": (dnn.VGG16ImageNet, 3, 224, 100),
+ "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 500),
+ "resnet18_cifar10": (dnn.ResNet18, 3, 32, 500),
+ "resnet50_imagenet": (dnn.ResNet50, 3, 224, 25),
+ "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 500),
+ "vgg16_cifar100": (dnn.VGG16Cifar100, 3, 32, 500),
+ "vgg16_imagenet": (dnn.VGG16ImageNet, 3, 224, 10),
}
self_folder = Path(__file__).parent
netname = argv[1]
@@ -44,6 +44,7 @@ bin_testset = BinDataset(
model: Module = model_cls()
checkpoint = self_folder / "../model_params/pytorch" / f"{netname}.pth.tar"
model.load_state_dict(torch.load(checkpoint.as_posix()))
+print(model)
build_dir = codegen_dir / "build"
target_binary = build_dir / netname
diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test
index 4010f21fc1..5544c75d28 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar10.test
@@ -1 +1,2 @@
RUN: test_frontend.py vgg16_cifar10
+RUN: check_dnn_acc.py final_accuracy vgg16_cifar10
diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test
index 23b43e6288..66bd69ee37 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_cifar100.test
@@ -1 +1,2 @@
RUN: test_frontend.py vgg16_cifar100
+RUN: check_dnn_acc.py final_accuracy vgg16_cifar100
diff --git a/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test b/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test
index e864c1e943..6529998ec4 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test
+++ b/hpvm/test/dnn_benchmarks/pytorch/vgg16_imagenet.test
@@ -1 +1,2 @@
RUN: test_frontend.py vgg16_imagenet
+RUN: check_dnn_acc.py final_accuracy vgg16_imagenet
--
GitLab