From 424a32a218908299d7510649d6174d2fdee77394 Mon Sep 17 00:00:00 2001
From: Yifan Zhao <yifanz16@illinois.edu>
Date: Sun, 4 Apr 2021 03:10:00 -0500
Subject: [PATCH] Moved profiling test cases to test frontend + profiling
---
hpvm/test/CMakeLists.txt | 3 +--
.../dnn_benchmarks/profiling/CMakeLists.txt | 15 -------------
.../profiling/alexnet2_cifar10.test | 1 -
.../profiling/alexnet_cifar10.test | 1 -
.../dnn_benchmarks/profiling/lenet_mnist.test | 1 -
.../profiling/mobilenet_cifar10.test | 1 -
.../profiling/resnet18_cifar10.test | 1 -
.../profiling/test_hpvm_c_profiling.py | 22 -------------------
.../dnn_benchmarks/pytorch/CMakeLists.txt | 16 ++++++++++++--
.../test_profiling/alexnet2_cifar10.test | 1 +
.../test_profiling/alexnet_cifar10.test | 1 +
.../pytorch/test_profiling/lenet_mnist.test | 1 +
.../test_profiling}/lit.cfg.py | 2 +-
.../test_profiling/mobilenet_cifar10.test | 1 +
.../test_profiling/resnet18_cifar10.test | 1 +
.../pytorch/test_profiling/test_profiling.py | 22 +++++++++++++++++++
.../pytorch/test_tuning/test_tuning.py | 7 ------
17 files changed, 43 insertions(+), 54 deletions(-)
delete mode 100644 hpvm/test/dnn_benchmarks/profiling/CMakeLists.txt
delete mode 100644 hpvm/test/dnn_benchmarks/profiling/alexnet2_cifar10.test
delete mode 100644 hpvm/test/dnn_benchmarks/profiling/alexnet_cifar10.test
delete mode 100644 hpvm/test/dnn_benchmarks/profiling/lenet_mnist.test
delete mode 100644 hpvm/test/dnn_benchmarks/profiling/mobilenet_cifar10.test
delete mode 100644 hpvm/test/dnn_benchmarks/profiling/resnet18_cifar10.test
delete mode 100755 hpvm/test/dnn_benchmarks/profiling/test_hpvm_c_profiling.py
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/test_profiling/alexnet2_cifar10.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/test_profiling/alexnet_cifar10.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/test_profiling/lenet_mnist.test
rename hpvm/test/dnn_benchmarks/{profiling => pytorch/test_profiling}/lit.cfg.py (91%)
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/test_profiling/mobilenet_cifar10.test
create mode 100644 hpvm/test/dnn_benchmarks/pytorch/test_profiling/resnet18_cifar10.test
create mode 100755 hpvm/test/dnn_benchmarks/pytorch/test_profiling/test_profiling.py
diff --git a/hpvm/test/CMakeLists.txt b/hpvm/test/CMakeLists.txt
index 4ff98a5386..8f8ca9b4c7 100644
--- a/hpvm/test/CMakeLists.txt
+++ b/hpvm/test/CMakeLists.txt
@@ -8,6 +8,5 @@ set(CLANG_CXX ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/clang++)
add_subdirectory(hpvm_pass) # Passes test suite
add_subdirectory(benchmarks)
add_subdirectory(dnn_benchmarks/hpvm-c) # HPVM-C DNN accuracy test suite
-add_subdirectory(dnn_benchmarks/pytorch) # Torch frontend test suite
+add_subdirectory(dnn_benchmarks/pytorch) # Torch frontend test suites (3 of them)
add_subdirectory(dnn_benchmarks/tensor-rt-src) # tensor_runtime DNN (build only, no tests)
-add_subdirectory(dnn_benchmarks/profiling) # hpvm-profiler test suite
diff --git a/hpvm/test/dnn_benchmarks/profiling/CMakeLists.txt b/hpvm/test/dnn_benchmarks/profiling/CMakeLists.txt
deleted file mode 100644
index 23e0e91618..0000000000
--- a/hpvm/test/dnn_benchmarks/profiling/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-# --[ llvm-lit test setup
-# lit.cfg.py looks for tests in CMAKE_CURRENT_BINARY_DIR (see lit.cfg.py)
-# as most of the tests require some kind of compilation / generation
-# which is best done over there.
-configure_lit_site_cfg(
- ../../lit.site.cfg.py.in
- ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
- MAIN_CONFIG
- ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
-)
-add_lit_testsuite(check-hpvm-profiler "Run tests for package hpvm-profiler"
- ${CMAKE_CURRENT_BINARY_DIR}
- DEPENDS dnn_benchmarks # Requires all dnn benchmarks
- ARGS "-j1" # Run DNN benchmarks sequentially
-)
diff --git a/hpvm/test/dnn_benchmarks/profiling/alexnet2_cifar10.test b/hpvm/test/dnn_benchmarks/profiling/alexnet2_cifar10.test
deleted file mode 100644
index 455a3e75a7..0000000000
--- a/hpvm/test/dnn_benchmarks/profiling/alexnet2_cifar10.test
+++ /dev/null
@@ -1 +0,0 @@
-RUN: test_hpvm_c_profiling.py alexnet2_cifar10
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/profiling/alexnet_cifar10.test b/hpvm/test/dnn_benchmarks/profiling/alexnet_cifar10.test
deleted file mode 100644
index 62c667a249..0000000000
--- a/hpvm/test/dnn_benchmarks/profiling/alexnet_cifar10.test
+++ /dev/null
@@ -1 +0,0 @@
-RUN: test_hpvm_c_profiling.py alexnet_cifar10
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/profiling/lenet_mnist.test b/hpvm/test/dnn_benchmarks/profiling/lenet_mnist.test
deleted file mode 100644
index 88856a8913..0000000000
--- a/hpvm/test/dnn_benchmarks/profiling/lenet_mnist.test
+++ /dev/null
@@ -1 +0,0 @@
-RUN: test_hpvm_c_profiling.py lenet_mnist
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/profiling/mobilenet_cifar10.test b/hpvm/test/dnn_benchmarks/profiling/mobilenet_cifar10.test
deleted file mode 100644
index a40981c940..0000000000
--- a/hpvm/test/dnn_benchmarks/profiling/mobilenet_cifar10.test
+++ /dev/null
@@ -1 +0,0 @@
-RUN: test_hpvm_c_profiling.py mobilenet_cifar10
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/profiling/resnet18_cifar10.test b/hpvm/test/dnn_benchmarks/profiling/resnet18_cifar10.test
deleted file mode 100644
index 5d09297309..0000000000
--- a/hpvm/test/dnn_benchmarks/profiling/resnet18_cifar10.test
+++ /dev/null
@@ -1 +0,0 @@
-RUN: test_hpvm_c_profiling.py resnet18_cifar10
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/profiling/test_hpvm_c_profiling.py b/hpvm/test/dnn_benchmarks/profiling/test_hpvm_c_profiling.py
deleted file mode 100755
index 853b0dc3e2..0000000000
--- a/hpvm/test/dnn_benchmarks/profiling/test_hpvm_c_profiling.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python3
-from pathlib import Path
-from sys import argv
-
-from hpvm_profiler import profile_configs, read_hpvm_configs, write_hpvm_configs
-
-# relative to cwd()
-benchmarks_bindir = Path("../hpvm-c")
-# relative to location of this file
-benchmarks_srcdir = Path(__file__).parent / "../hpvm-c/benchmarks"
-# We're called in the "current" binary directory.
-# For example (depending on where build dir is),
-# "hpvm/build/tools/hpvm/test/dnn_benchmarks/profiling".
-# So we know where the benchmark binaries are due to source directory structure,
-# and this is not hardcoding.
-dnn = argv[1]
-bench_bin_file = benchmarks_bindir / f"hpvm_{dnn}"
-config_file = benchmarks_srcdir / dnn / "data/tuner_confs.txt"
-out_config_file = f"./{dnn}.txt"
-header, configs = read_hpvm_configs(config_file)
-profile_configs(bench_bin_file, configs[1:6], configs[0], progress_bar=False)
-write_hpvm_configs(header, configs[:6], out_config_file)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt
index 690c62f387..09ceddfb46 100644
--- a/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt
+++ b/hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt
@@ -5,7 +5,7 @@ configure_lit_site_cfg(
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/test_frontend/lit.cfg.py
)
-add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for HPVM PyTorch frontend"
+add_lit_testsuite(check-hpvm-torch2hpvm "Run accuracy tests for HPVM PyTorch frontend"
${CMAKE_CURRENT_BINARY_DIR}/test_frontend
# We depend on check_dnn_acc.py defined in ../hpvm-c/
# to compare the inference accuracy of our frontend-generated binary
@@ -14,6 +14,18 @@ add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for HPVM PyTorch frontend"
ARGS "-j1" # Run frontend generation sequentially
)
+# --[ llvm-lit test setup for test_profiling/
+configure_lit_site_cfg(
+ ../../lit.site.cfg.py.in
+ ${CMAKE_CURRENT_BINARY_DIR}/test_profiling/lit.site.cfg.py
+ MAIN_CONFIG
+ ${CMAKE_CURRENT_SOURCE_DIR}/test_profiling/lit.cfg.py
+)
+add_lit_testsuite(check-hpvm-profiling "Run tests for frontend+profiling"
+ ${CMAKE_CURRENT_BINARY_DIR}/test_profiling
+ ARGS "-j1" # Run DNN benchmarks sequentially
+)
+
# --[ llvm-lit test setup for test_tuning/
configure_lit_site_cfg(
../../lit.site.cfg.py.in
@@ -21,7 +33,7 @@ configure_lit_site_cfg(
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/test_tuning/lit.cfg.py
)
-add_lit_testsuite(check-hpvm-tuning "Run tests for autotuning procedure"
+add_lit_testsuite(check-hpvm-tuning "Run tests for frontend+autotuning"
${CMAKE_CURRENT_BINARY_DIR}/test_tuning
ARGS "-j1" # Run tuning tests sequentially
)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_profiling/alexnet2_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/alexnet2_cifar10.test
new file mode 100644
index 0000000000..f544576dcf
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/alexnet2_cifar10.test
@@ -0,0 +1 @@
+RUN: test_profiling.py alexnet2_cifar10
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_profiling/alexnet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/alexnet_cifar10.test
new file mode 100644
index 0000000000..af40686e9a
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/alexnet_cifar10.test
@@ -0,0 +1 @@
+RUN: test_profiling.py alexnet_cifar10
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_profiling/lenet_mnist.test b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/lenet_mnist.test
new file mode 100644
index 0000000000..f556f9adb0
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/lenet_mnist.test
@@ -0,0 +1 @@
+RUN: test_profiling.py lenet_mnist
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/profiling/lit.cfg.py b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/lit.cfg.py
similarity index 91%
rename from hpvm/test/dnn_benchmarks/profiling/lit.cfg.py
rename to hpvm/test/dnn_benchmarks/pytorch/test_profiling/lit.cfg.py
index 5c11f61baf..95ae7c44bf 100644
--- a/hpvm/test/dnn_benchmarks/profiling/lit.cfg.py
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/lit.cfg.py
@@ -29,4 +29,4 @@ config.test_exec_root = current_binary_dir
llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
# Add substitution for our main script in this directory.
-llvm_config.add_tool_substitutions(["test_hpvm_c_profiling.py"], config.test_source_root)
+llvm_config.add_tool_substitutions(["test_profiling.py"], config.test_source_root)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_profiling/mobilenet_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/mobilenet_cifar10.test
new file mode 100644
index 0000000000..18276e5e79
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/mobilenet_cifar10.test
@@ -0,0 +1 @@
+RUN: test_profiling.py mobilenet_cifar10
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_profiling/resnet18_cifar10.test b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/resnet18_cifar10.test
new file mode 100644
index 0000000000..676e80393b
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/resnet18_cifar10.test
@@ -0,0 +1 @@
+RUN: test_profiling.py resnet18_cifar10
\ No newline at end of file
diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_profiling/test_profiling.py b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/test_profiling.py
new file mode 100755
index 0000000000..66aa33d7cb
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_profiling/test_profiling.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+import shutil
+import site
+from pathlib import Path
+from sys import argv
+
+from hpvm_profiler import profile_configs, read_hpvm_configs, write_hpvm_configs
+
+self_folder = Path(__file__).parent.absolute()
+site.addsitedir(self_folder.parent)
+import dnn
+
+netname = argv[1]
+codegen_dir = Path(f"./{netname}")
+if codegen_dir.exists():
+ shutil.rmtree(codegen_dir)
+binary_file, _ = dnn.export_example_dnn(netname, codegen_dir, False)
+config_file = self_folder / "../../hpvm-c/benchmarks" / netname / "data/tuner_confs.txt"
+out_config_file = f"./{netname}.txt"
+header, configs = read_hpvm_configs(config_file)
+profile_configs(binary_file, configs[1:6], configs[0], progress_bar=False)
+write_hpvm_configs(header, configs[:6], out_config_file)
diff --git a/hpvm/test/dnn_benchmarks/pytorch/test_tuning/test_tuning.py b/hpvm/test/dnn_benchmarks/pytorch/test_tuning/test_tuning.py
index 746a6d8679..c589b108d7 100755
--- a/hpvm/test/dnn_benchmarks/pytorch/test_tuning/test_tuning.py
+++ b/hpvm/test/dnn_benchmarks/pytorch/test_tuning/test_tuning.py
@@ -14,13 +14,6 @@ import dnn
msg_logger = config_pylogger(output_dir=".", verbose=True)
-def generate(model_cls, nch, img_size, batch_size, netname):
- codegen_dir = Path(f"./{netname}")
- binary_file, exporter = dnn.export_example_dnn(netname, codegen_dir, True)
- metadata_file = codegen_dir / exporter.metadata_file
- return binary_file, metadata_file
-
-
def main():
netname, is_pred = argv[1:]
is_pred = int(is_pred)
--
GitLab