Skip to content
Snippets Groups Projects
Commit adadde87 authored by akashk4's avatar akashk4
Browse files

Add scripts to run benchmarks

parent 8c69ad52
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
hpvm/test/dnn_benchmarks/scripts/run_dnn.py
+ 156
− 81
View file @ adadde87
...@@ -4,6 +4,10 @@ import sys ...@@ -4,6 +4,10 @@ import sys
binary_dir = "../../../build/tools/hpvm/test/dnn_benchmarks/" binary_dir = "../../../build/tools/hpvm/test/dnn_benchmarks/"
accuracy_file = "final_accuracy"
profile_file = "profile_data.txt"
profile_file_prefix = "profile_info_"
temp_file_name = "temp.txt" temp_file_name = "temp.txt"
pred_binary_prefix = "test_" pred_binary_prefix = "test_"
pred_binary_suffix = "_pred" pred_binary_suffix = "_pred"
...@@ -15,67 +19,141 @@ max_num_runs = 20 ...@@ -15,67 +19,141 @@ max_num_runs = 20
def max_num_configs (config_file): def max_num_configs (config_file):
num_configs = 0 num_configs = 0
with open(config_file, "r") as f: with open(config_file, "r") as f:
for line in f: for line in f:
if "conf" in line: if "conf" in line:
num_configs = num_configs + 1 num_configs = num_configs + 1
return (num_configs + 1) return (num_configs + 1)
def read_and_write_config (config_file, config_num, temp_file): def read_and_write_config (config_file, config_num, temp_file):
config = "" config = ""
print("--CONFIG FILE: " + config_file)
print("--CONFIG NUM: " + str(config_num))
print("--TEMP FILE: " + temp_file)
with open(config_file, "r") as f: with open(config_file, "r") as f:
conf = "conf" + str(config_num) conf = "conf" + str(config_num)
read_config = False read_config = False
read_first_line = False read_first_line = False
for line in f: for line in f:
if read_first_line == False: if read_first_line == False:
config = config + line config = config + line
read_first_line = True read_first_line = True
continue continue
if "-----" in line and read_config == True: if "-----" in line and read_config == True:
read_config = False read_config = False
config = config + line config = config + line
break break
if read_config == True: if read_config == True:
config = config + line config = config + line
continue continue
if conf in line: if conf in line:
read_config = True read_config = True
config = config + "+++++\n" config = config + "+++++\n"
config = config + line config = config + line
print("config: ") print("config: ")
print(config) print(config)
with open(temp_file, "w") as f: with open(temp_file, "w") as f:
f.write(config) f.write(config)
def predictive_tuning_exec(): def get_avg_exec_time(profile_file_path, config_num):
num_args = len(sys.argv) prof_file = profile_file_path + profile_file_prefix + str(config_num) + ".txt"
binary_files = list() print("PROFILE FILE: " + prof_file)
arg = 2 with open(prof_file, "r") as f:
while arg < num_args: for line in f:
binary_files.append(sys.argv[arg]) if "Total Time" in line:
arg = arg + 1 print("LINE: " + line)
time = line.strip().split() [3]
for dnn_name in binary_files: print("TIME: " + time)
dnn_dir = "../benchmarks/" + dnn_name return float(time)
binary_name = binary_dir + pred_binary_prefix + dnn_name + pred_binary_suffix print("ERROR")
pred_dir = dnn_dir + "/predictive" sys.exit()
config_file = pred_dir + "/" + dnn_name + ".txt" return float(-1)
temp_file = pred_dir + "/" + temp_file_name
print("dnn_dir: " + dnn_dir) def get_exec_time(config_file):
print("binary name: " + binary_name) print("CONFIG FILE: " + config_file)
print("pred_dir: " + pred_dir) with open(config_file, "r") as f:
print("config_file: " + config_file) for line in f:
print("temp_file: " + temp_file) if "conf" in line:
config_num = 1 print("LINE: " + line)
max_configs = max_num_configs(config_file) time = line.strip().split() [1]
while config_num < max_configs: print("TIME: " + time)
read_and_write_config(config_file, config_num, temp_file) return float(time)
exec_command = binary_name print("ERROR")
print(exec_command) sys.exit()
os.system(exec_command) return float(-1)
config_num = config_num + 1
def get_avg_exec_accuracy(file_name):
with open(file_name, "r") as f:
for line in f:
accuracy = line.strip().split() [0]
print("ACCURACY: " + accuracy)
return float(accuracy)
print("ERROR")
sys.exit()
return float(-1)
def get_exec_accuracy(config_file):
with open(config_file, "r") as f:
for line in f:
if "conf" in line:
print("LINE: " + line)
acc = line.strip().split() [4]
print("ACCURACY: " + acc)
return float(acc)
print("ERROR")
sys.exit()
return float(-1)
def predictive_tuning_exec(dnn_name):
#num_args = len(sys.argv)
#binary_files = list()
#arg = 2
#while arg < num_args:
# binary_files.append(sys.argv[arg])
# arg = arg + 1
#for dnn_name in binary_files:
dnn_dir = "../benchmarks/" + dnn_name
binary_name = binary_dir + pred_binary_prefix + dnn_name + pred_binary_suffix
pred_dir = dnn_dir + "/predictive/"
config_file = pred_dir + dnn_name + ".txt"
temp_file = pred_dir + temp_file_name
print("dnn_dir: " + dnn_dir)
print("binary name: " + binary_name)
print("pred_dir: " + pred_dir)
print("config_file: " + config_file)
print("temp_file: " + temp_file)
config_num = 1
max_configs = max_num_configs(config_file)
baseline_time = 0
baseline_acc = 0
print("MAX CONFIGS: " + str(max_configs))
while config_num < max_configs:
read_and_write_config(config_file, config_num, temp_file)
exec_command = binary_name
print(exec_command)
os.system(exec_command)
time = get_avg_exec_time(pred_dir, config_num - 1)
acc = get_avg_exec_accuracy(accuracy_file)
config_time = get_exec_time(temp_file)
config_acc = get_exec_accuracy(temp_file)
if config_num == 1:
baseline_time = time
baseline_acc = acc
else:
print("SPEEDUP: ")
print(baseline_time/time)
print("CONFIG TIME: ")
print(config_time)
print("ACC LOSS: ")
print(baseline_acc - acc)
print("CONFIG ACC: ")
print(config_acc)
config_num = config_num + 1
exec_command = "rm " + temp_file + " " + accuracy_file + " " + profile_file + " " + pred_dir + "profile*"
print(exec_command)
os.system(exec_command)
def runtime_tuning_exec(): def runtime_tuning_exec():
...@@ -87,38 +165,35 @@ def runtime_tuning_exec(): ...@@ -87,38 +165,35 @@ def runtime_tuning_exec():
arg = arg + 1 arg = arg + 1
for dnn_name in binary_files: for dnn_name in binary_files:
binary_dir = "../benchmarks/" + dnn_name binary_dir = "../benchmarks/" + dnn_name
binary_name = binary_dir + rt_binary_suffix binary_name = binary_dir + rt_binary_suffix
conf_dir = binary_dir + "/data" conf_dir = binary_dir + "/data"
print("binary_dir: " + binary_dir) print("binary_dir: " + binary_dir)
print("binary name: " + binary_name) print("binary name: " + binary_name)
run = 0 run = 0
while run < max_num_runs: while run < max_num_runs:
exec_command = binary_name exec_command = binary_name
print(exec_command) print(exec_command)
os.system(exec_command) os.system(exec_command)
exec_command = "/home/nvidia/poll 13" exec_command = "/home/nvidia/poll 13"
print(exec_command) print(exec_command)
os.system(exec_command) os.system(exec_command)
exec_command = "mv " + conf_dir + "/profile_info_0.txt " + conf_dir + "/profile_info_out-run-" + str(run) + ".txt" exec_command = "mv " + conf_dir + "/profile_info_0.txt " + conf_dir + "/profile_info_out-run-" + str(run) + ".txt"
print(exec_command) print(exec_command)
os.system(exec_command) os.system(exec_command)
run = run + 1 run = run + 1
exec_command = "rm -rf " + conf_dir + "/run_data" exec_command = "rm -rf " + conf_dir + "/run_data"
print(exec_command) print(exec_command)
os.system(exec_command) os.system(exec_command)
exec_command = "mkdir " + conf_dir + "/run_data" exec_command = "mkdir " + conf_dir + "/run_data"
print(exec_command) print(exec_command)
os.system(exec_command) os.system(exec_command)
if __name__ == "__main__": if __name__ == "__main__":
if sys.argv[1] == "--runtime_tuning": if sys.argv[1] == "--runtime_tuning":
runtime_tuning_exec() runtime_tuning_exec()
elif sys.argv[1] == "--predictive_tuning":
predictive_tuning_exec()
else: else:
print("Error: Use --runtime_tuning or --predictive_tuning flags!") predictive_tuning_exec(sys.argv[1])
sys.exit()
hpvm/test/dnn_benchmarks/scripts/run_dnns.py
+ 13
− 13
View file @ adadde87
import os import os
import sys import sys
#dnns = "alexnet alexnet2 vgg16_cifar10 vgg16_cifar100 resnet18 mobilenet lenet_mnist" #dnns = ["alexnet", "alexnet2", "vgg16_cifar10", "vgg16_cifar100", "resnet18", "mobilenet", "lenet_mnist"]
dnns = "alexnet" dnns = ["resnet18"]
#if sys.argv[1] == "--runtime":
# exec_command = "python3 run_dnn.py" + " --runtime_tuning " + dnns
# print(exec_command)
# os.system(exec_command)
#else:
if __name__ == "__main__":
for dnn in dnns:
exec_command = "python3 run_dnn.py " + dnn
print(exec_command)
os.system(exec_command)
if sys.argv[1] == "--runtime":
exec_command = "python3 run_dnn.py" + " --runtime_tuning " + dnns
print(exec_command)
os.system(exec_command)
elif sys.argv[1] == "--predictive":
exec_command = "python3 run_dnn.py" + " --predictive_tuning " + dnns
print(exec_command)
os.system(exec_command)
else:
print("Error: Use --runtime_tuning or --predictive_tuning flags!")
sys.exit()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment