From 7fd1611c0dc50b507ec2b9026b4e5fde146af193 Mon Sep 17 00:00:00 2001
From: Akash Kothari <akashk4@tyler.cs.illinois.edu>
Date: Fri, 5 Feb 2021 00:04:06 -0600
Subject: [PATCH] Update scripts to run CNNs and add README
---
.../dnn_benchmarks/hpvm-c/scripts/README.md | 16 +++++++
.../{ => hpvm-c}/scripts/run_dnn.py | 46 +++++++++----------
.../{ => hpvm-c}/scripts/run_dnns.py | 7 +--
3 files changed, 39 insertions(+), 30 deletions(-)
create mode 100644 hpvm/test/dnn_benchmarks/hpvm-c/scripts/README.md
rename hpvm/test/dnn_benchmarks/{ => hpvm-c}/scripts/run_dnn.py (87%)
rename hpvm/test/dnn_benchmarks/{ => hpvm-c}/scripts/run_dnns.py (64%)
diff --git a/hpvm/test/dnn_benchmarks/hpvm-c/scripts/README.md b/hpvm/test/dnn_benchmarks/hpvm-c/scripts/README.md
new file mode 100644
index 0000000000..bcde5c03ab
--- /dev/null
+++ b/hpvm/test/dnn_benchmarks/hpvm-c/scripts/README.md
@@ -0,0 +1,16 @@
+## Running CNN benchmarks
+
+We provide 2 scripts to run the CNN benchmarks automatically: run_dnn.py and run_dnns.py.
+
+In order to run all CNN benchmarks, execute the following:
+```
+python run_dnns.py
+```
+
+And to run a particular benchmark, one must specify the name of the benchmark:
+```
+python run_dnn.py <dnn_name>
+```
+
+These scripts not only automate the execution of the benchmarks, but also produce trade-off curves between accuracy and performance for executed benchmarks. These trade-off curves are placed under ./hpvm/hpvm/docs/tradeoff-curves. Trade-off curves for the CNN benchmarks suit are already provided as examples.
+
diff --git a/hpvm/test/dnn_benchmarks/scripts/run_dnn.py b/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnn.py
similarity index 87%
rename from hpvm/test/dnn_benchmarks/scripts/run_dnn.py
rename to hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnn.py
index 2eed6739a7..10eaf5699f 100644
--- a/hpvm/test/dnn_benchmarks/scripts/run_dnn.py
+++ b/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnn.py
@@ -1,18 +1,20 @@
import os.path
from os import path
import sys
-#import matplotlib.pyplot as plt
+import matplotlib.pyplot as plt
binary_dir = "../../../build/tools/hpvm/test/dnn_benchmarks/"
+tradeoff_curves_dir = "../../../docs/tradeoff-curves"
+
accuracy_file = "final_accuracy"
profile_file = "profile_data.txt"
profile_file_prefix = "profile_info_"
temp_file_name = "temp.txt"
-pred_binary_prefix = "test_"
-pred_binary_suffix = "_pred"
+pred_binary_prefix = ""
+pred_binary_suffix = ""
rt_binary_suffix = "_rt_pred"
max_num_runs = 20
@@ -108,14 +110,6 @@ def get_exec_accuracy(config_file):
return float(-1)
def predictive_tuning_exec(dnn_name):
- #num_args = len(sys.argv)
- #binary_files = list()
- #arg = 2
- #while arg < num_args:
- # binary_files.append(sys.argv[arg])
- # arg = arg + 1
-
- #for dnn_name in binary_files:
dnn_dir = "../benchmarks/" + dnn_name
binary_name = binary_dir + pred_binary_prefix + dnn_name + pred_binary_suffix
pred_dir = dnn_dir + "/predictive/"
@@ -151,22 +145,29 @@ def predictive_tuning_exec(dnn_name):
else:
print("SPEEDUP: ")
print(baseline_time/time)
- #time.append(baseline_time/time)
+ perf_list.append(baseline_time/time)
print("CONFIG TIME: ")
print(config_time)
print("ACC LOSS: ")
print(baseline_acc - acc)
- #acc_list.append(baseline_acc - acc)
+ acc_list.append(baseline_acc - acc)
print("CONFIG ACC: ")
print(config_acc)
config_num = config_num + 1
- #plt.plot(perf_list, acc_list)
- #plt.xlabel("Speedups")
- #plt.ylabel("Accurancy loss")
- #plt.savefig(pred_dir + "tradeoff.pdf")
- #exec_command = "rm " + temp_file + " " + accuracy_file + " " + profile_file + " " + pred_dir + "profile*"
- #print(exec_command)
- #os.system(exec_command)
+ exec_command = "rm " + temp_file + " " + accuracy_file + " " + profile_file + " " + pred_dir + "profile*"
+ print(exec_command)
+ os.system(exec_command)
+ plt.scatter(acc_list, perf_list)
+ plt.ylabel("Speedup (X)")
+ plt.xlabel("Accurancy loss (%)")
+ xticks = ['-1', '0', '1', '2', '3', '4']
+ yticks = ['1', '1.5', '2', '2.5', '3']
+ plt.xlim(-1, 4)
+ plt.ylim(1, 3)
+ plt.title(dnn_name)
+ plt.savefig(tradeoff_curves_dir + dnn_name + "_tradeoff.pdf")
+ perf_list.clear()
+ acc_list.clear()
def runtime_tuning_exec():
@@ -205,8 +206,5 @@ def runtime_tuning_exec():
if __name__ == "__main__":
- if sys.argv[1] == "--runtime_tuning":
- runtime_tuning_exec()
- else:
- predictive_tuning_exec(sys.argv[1])
+ predictive_tuning_exec(sys.argv[1])
diff --git a/hpvm/test/dnn_benchmarks/scripts/run_dnns.py b/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnns.py
similarity index 64%
rename from hpvm/test/dnn_benchmarks/scripts/run_dnns.py
rename to hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnns.py
index 0de85c7847..4c9ab6ec12 100644
--- a/hpvm/test/dnn_benchmarks/scripts/run_dnns.py
+++ b/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnns.py
@@ -2,13 +2,8 @@ import os
import sys
dnns = ["alexnet", "alexnet2", "vgg16_cifar10", "vgg16_cifar100", "resnet18", "mobilenet_cifar10", "alexnet_imagenet", "resnet50_imagenet", "vgg16_imagenet", "lenet_mnist"]
-#dnns = ["resnet50_imagenet","alexnet"]
-#if sys.argv[1] == "--runtime":
-# exec_command = "python3 run_dnn.py" + " --runtime_tuning " + dnns
-# print(exec_command)
-# os.system(exec_command)
-#else:
+
if __name__ == "__main__":
for dnn in dnns:
exec_command = "python3 run_dnn.py " + dnn
--
GitLab