From adadde87fdad62671fee41b943e2d495ebdae8b1 Mon Sep 17 00:00:00 2001
From: akashk4 <hashim.sharif91@gmail.com>
Date: Sat, 30 Jan 2021 11:03:45 -0600
Subject: [PATCH] Add scripts to run benchmarks
---
hpvm/test/dnn_benchmarks/scripts/run_dnn.py | 237 ++++++++++++-------
hpvm/test/dnn_benchmarks/scripts/run_dnns.py | 26 +-
2 files changed, 169 insertions(+), 94 deletions(-)
diff --git a/hpvm/test/dnn_benchmarks/scripts/run_dnn.py b/hpvm/test/dnn_benchmarks/scripts/run_dnn.py
index b4e4aa2b2b..db1a5bbb35 100644
--- a/hpvm/test/dnn_benchmarks/scripts/run_dnn.py
+++ b/hpvm/test/dnn_benchmarks/scripts/run_dnn.py
@@ -4,6 +4,10 @@ import sys
binary_dir = "../../../build/tools/hpvm/test/dnn_benchmarks/"
+accuracy_file = "final_accuracy"
+profile_file = "profile_data.txt"
+profile_file_prefix = "profile_info_"
+
temp_file_name = "temp.txt"
pred_binary_prefix = "test_"
pred_binary_suffix = "_pred"
@@ -15,67 +19,141 @@ max_num_runs = 20
def max_num_configs (config_file):
num_configs = 0
with open(config_file, "r") as f:
- for line in f:
- if "conf" in line:
- num_configs = num_configs + 1
+ for line in f:
+ if "conf" in line:
+ num_configs = num_configs + 1
return (num_configs + 1)
def read_and_write_config (config_file, config_num, temp_file):
config = ""
+ print("--CONFIG FILE: " + config_file)
+ print("--CONFIG NUM: " + str(config_num))
+ print("--TEMP FILE: " + temp_file)
with open(config_file, "r") as f:
- conf = "conf" + str(config_num)
- read_config = False
- read_first_line = False
- for line in f:
- if read_first_line == False:
- config = config + line
- read_first_line = True
- continue
- if "-----" in line and read_config == True:
- read_config = False
- config = config + line
- break
- if read_config == True:
- config = config + line
- continue
- if conf in line:
- read_config = True
- config = config + "+++++\n"
- config = config + line
+ conf = "conf" + str(config_num)
+ read_config = False
+ read_first_line = False
+ for line in f:
+ if read_first_line == False:
+ config = config + line
+ read_first_line = True
+ continue
+ if "-----" in line and read_config == True:
+ read_config = False
+ config = config + line
+ break
+ if read_config == True:
+ config = config + line
+ continue
+ if conf in line:
+ read_config = True
+ config = config + "+++++\n"
+ config = config + line
print("config: ")
print(config)
with open(temp_file, "w") as f:
- f.write(config)
-
-
-def predictive_tuning_exec():
- num_args = len(sys.argv)
- binary_files = list()
- arg = 2
- while arg < num_args:
- binary_files.append(sys.argv[arg])
- arg = arg + 1
-
- for dnn_name in binary_files:
- dnn_dir = "../benchmarks/" + dnn_name
- binary_name = binary_dir + pred_binary_prefix + dnn_name + pred_binary_suffix
- pred_dir = dnn_dir + "/predictive"
- config_file = pred_dir + "/" + dnn_name + ".txt"
- temp_file = pred_dir + "/" + temp_file_name
- print("dnn_dir: " + dnn_dir)
- print("binary name: " + binary_name)
- print("pred_dir: " + pred_dir)
- print("config_file: " + config_file)
- print("temp_file: " + temp_file)
- config_num = 1
- max_configs = max_num_configs(config_file)
- while config_num < max_configs:
- read_and_write_config(config_file, config_num, temp_file)
- exec_command = binary_name
- print(exec_command)
- os.system(exec_command)
- config_num = config_num + 1
+ f.write(config)
+
+
+def get_avg_exec_time(profile_file_path, config_num):
+ prof_file = profile_file_path + profile_file_prefix + str(config_num) + ".txt"
+ print("PROFILE FILE: " + prof_file)
+ with open(prof_file, "r") as f:
+ for line in f:
+ if "Total Time" in line:
+ print("LINE: " + line)
+ time = line.strip().split() [3]
+ print("TIME: " + time)
+ return float(time)
+ print("ERROR")
+ sys.exit()
+ return float(-1)
+
+def get_exec_time(config_file):
+ print("CONFIG FILE: " + config_file)
+ with open(config_file, "r") as f:
+ for line in f:
+ if "conf" in line:
+ print("LINE: " + line)
+ time = line.strip().split() [1]
+ print("TIME: " + time)
+ return float(time)
+ print("ERROR")
+ sys.exit()
+ return float(-1)
+
+def get_avg_exec_accuracy(file_name):
+ with open(file_name, "r") as f:
+ for line in f:
+ accuracy = line.strip().split() [0]
+ print("ACCURACY: " + accuracy)
+ return float(accuracy)
+ print("ERROR")
+ sys.exit()
+ return float(-1)
+
+def get_exec_accuracy(config_file):
+ with open(config_file, "r") as f:
+ for line in f:
+ if "conf" in line:
+ print("LINE: " + line)
+ acc = line.strip().split() [4]
+ print("ACCURACY: " + acc)
+ return float(acc)
+ print("ERROR")
+ sys.exit()
+ return float(-1)
+
+def predictive_tuning_exec(dnn_name):
+ #num_args = len(sys.argv)
+ #binary_files = list()
+ #arg = 2
+ #while arg < num_args:
+ # binary_files.append(sys.argv[arg])
+ # arg = arg + 1
+
+ #for dnn_name in binary_files:
+ dnn_dir = "../benchmarks/" + dnn_name
+ binary_name = binary_dir + pred_binary_prefix + dnn_name + pred_binary_suffix
+ pred_dir = dnn_dir + "/predictive/"
+ config_file = pred_dir + dnn_name + ".txt"
+ temp_file = pred_dir + temp_file_name
+ print("dnn_dir: " + dnn_dir)
+ print("binary name: " + binary_name)
+ print("pred_dir: " + pred_dir)
+ print("config_file: " + config_file)
+ print("temp_file: " + temp_file)
+ config_num = 1
+ max_configs = max_num_configs(config_file)
+ baseline_time = 0
+ baseline_acc = 0
+ print("MAX CONFIGS: " + str(max_configs))
+ while config_num < max_configs:
+ read_and_write_config(config_file, config_num, temp_file)
+ exec_command = binary_name
+ print(exec_command)
+ os.system(exec_command)
+ time = get_avg_exec_time(pred_dir, config_num - 1)
+ acc = get_avg_exec_accuracy(accuracy_file)
+ config_time = get_exec_time(temp_file)
+ config_acc = get_exec_accuracy(temp_file)
+ if config_num == 1:
+ baseline_time = time
+ baseline_acc = acc
+ else:
+ print("SPEEDUP: ")
+ print(baseline_time/time)
+ print("CONFIG TIME: ")
+ print(config_time)
+ print("ACC LOSS: ")
+ print(baseline_acc - acc)
+ print("CONFIG ACC: ")
+ print(config_acc)
+ config_num = config_num + 1
+ exec_command = "rm " + temp_file + " " + accuracy_file + " " + profile_file + " " + pred_dir + "profile*"
+ print(exec_command)
+ os.system(exec_command)
def runtime_tuning_exec():
@@ -87,38 +165,35 @@ def runtime_tuning_exec():
arg = arg + 1
for dnn_name in binary_files:
- binary_dir = "../benchmarks/" + dnn_name
- binary_name = binary_dir + rt_binary_suffix
- conf_dir = binary_dir + "/data"
- print("binary_dir: " + binary_dir)
- print("binary name: " + binary_name)
- run = 0
- while run < max_num_runs:
- exec_command = binary_name
- print(exec_command)
- os.system(exec_command)
- exec_command = "/home/nvidia/poll 13"
- print(exec_command)
- os.system(exec_command)
- exec_command = "mv " + conf_dir + "/profile_info_0.txt " + conf_dir + "/profile_info_out-run-" + str(run) + ".txt"
- print(exec_command)
- os.system(exec_command)
- run = run + 1
- exec_command = "rm -rf " + conf_dir + "/run_data"
- print(exec_command)
- os.system(exec_command)
- exec_command = "mkdir " + conf_dir + "/run_data"
- print(exec_command)
- os.system(exec_command)
-
+ binary_dir = "../benchmarks/" + dnn_name
+ binary_name = binary_dir + rt_binary_suffix
+ conf_dir = binary_dir + "/data"
+ print("binary_dir: " + binary_dir)
+ print("binary name: " + binary_name)
+ run = 0
+ while run < max_num_runs:
+ exec_command = binary_name
+ print(exec_command)
+ os.system(exec_command)
+ exec_command = "/home/nvidia/poll 13"
+ print(exec_command)
+ os.system(exec_command)
+ exec_command = "mv " + conf_dir + "/profile_info_0.txt " + conf_dir + "/profile_info_out-run-" + str(run) + ".txt"
+ print(exec_command)
+ os.system(exec_command)
+ run = run + 1
+ exec_command = "rm -rf " + conf_dir + "/run_data"
+ print(exec_command)
+ os.system(exec_command)
+ exec_command = "mkdir " + conf_dir + "/run_data"
+ print(exec_command)
+ os.system(exec_command)
+
if __name__ == "__main__":
if sys.argv[1] == "--runtime_tuning":
- runtime_tuning_exec()
- elif sys.argv[1] == "--predictive_tuning":
- predictive_tuning_exec()
+ runtime_tuning_exec()
else:
- print("Error: Use --runtime_tuning or --predictive_tuning flags!")
- sys.exit()
+ predictive_tuning_exec(sys.argv[1])
diff --git a/hpvm/test/dnn_benchmarks/scripts/run_dnns.py b/hpvm/test/dnn_benchmarks/scripts/run_dnns.py
index 990fd67246..e827f6bc82 100644
--- a/hpvm/test/dnn_benchmarks/scripts/run_dnns.py
+++ b/hpvm/test/dnn_benchmarks/scripts/run_dnns.py
@@ -1,17 +1,17 @@
import os
import sys
-#dnns = "alexnet alexnet2 vgg16_cifar10 vgg16_cifar100 resnet18 mobilenet lenet_mnist"
-dnns = "alexnet"
+#dnns = ["alexnet", "alexnet2", "vgg16_cifar10", "vgg16_cifar100", "resnet18", "mobilenet", "lenet_mnist"]
+dnns = ["resnet18"]
+
+#if sys.argv[1] == "--runtime":
+# exec_command = "python3 run_dnn.py" + " --runtime_tuning " + dnns
+# print(exec_command)
+# os.system(exec_command)
+#else:
+if __name__ == "__main__":
+ for dnn in dnns:
+ exec_command = "python3 run_dnn.py " + dnn
+ print(exec_command)
+ os.system(exec_command)
-if sys.argv[1] == "--runtime":
- exec_command = "python3 run_dnn.py" + " --runtime_tuning " + dnns
- print(exec_command)
- os.system(exec_command)
-elif sys.argv[1] == "--predictive":
- exec_command = "python3 run_dnn.py" + " --predictive_tuning " + dnns
- print(exec_command)
- os.system(exec_command)
-else:
- print("Error: Use --runtime_tuning or --predictive_tuning flags!")
- sys.exit()
--
GitLab