diff --git a/llvm/projects/hpvm-tensor-rt/bin/tuner_src/benchmarks.py b/llvm/projects/hpvm-tensor-rt/bin/tuner_src/benchmarks.py index 7e969271c20031dab9f302b333a4f7feb0338871..41bd4c73483e19472053406104f8590e54bd5d36 100644 --- a/llvm/projects/hpvm-tensor-rt/bin/tuner_src/benchmarks.py +++ b/llvm/projects/hpvm-tensor-rt/bin/tuner_src/benchmarks.py @@ -12,10 +12,18 @@ # Batch 18: Batch13 (Basline) + ParetoCurve (1500 Runs) - BUGGY IGNORE!!! # Batch 19: (Basline) + ParetoCurve + 2 runs in Tuning Phase (1500 Runs) + # Batch 20: 3-Skip levels + + 2 runs + 1500 Runs + EnergyBandSize now % of Max (Compare against Batch19 -batch_id = "batch201" +# Batch 200: AlgoTuner - 1000 images - 1500 runs (IGNORE) +# Batch 201: AlgoTuner - 2000 images - 1500 runs +# Batch 202: AlgoTuner - 2000 images - 500 runs + + + +batch_id = "batch202" + class Benchmark: def __init__(self): @@ -65,7 +73,7 @@ Alexnet1.layer_knobs = "../opentuner/data/alexnet/knobs.txt" Alexnet1.loss1_result_file = "tuner_results/alexnet_cifar10/loss_1/promise_tuned_confs/promise_confs.txt" Alexnet1.loss2_result_file = "tuner_results/alexnet_cifar10/loss_2/promise_tuned_confs/promise_confs.txt" -Alexnet1.autotuner_runs = 1500 +Alexnet1.autotuner_runs = 500 Alexnet1.tuner_accuracy = 79.9 #Alexnet1.promise_accuracy = 79.9 Alexnet1.promise_accuracy = 79.5 @@ -96,7 +104,7 @@ Alexnet2.cost_file = "../build_tuner/tuner_results/alexnet2_cifar10/op_cost.txt" Alexnet2.layer_knobs = "../opentuner/data/alexnet2/knobs.txt" #Alexnet2.loss1_result_file = "tuner_results/alexnet2_cifar10/loss_1/promise_tuned_confs/promise_confs.txt" #Alexnet2.loss2_result_file = "tuner_results/alexnet2_cifar10/loss_2/promise_tuned_confs/promise_confs.txt" -Alexnet2.autotuner_runs = 1500 +Alexnet2.autotuner_runs = 500 Alexnet2.tuner_accuracy = 84.19 #Alexnet2.promise_accuracy = 84.19 Alexnet2.promise_accuracy = 84.8 @@ -131,7 +139,7 @@ Alexnet3.layer_knobs = "../opentuner/data/vgg16_cifar10/knobs.txt" Alexnet3.loss1_result_file = "tuner_results/vgg16_cifar10/loss_1/promise_tuned_confs/promise_confs.txt" Alexnet3.loss2_result_file = "tuner_results/vgg16_cifar10/loss_2/promise_tuned_confs/promise_confs.txt" -Alexnet3.autotuner_runs = 1500 +Alexnet3.autotuner_runs = 500 Alexnet3.tuner_accuracy = 90.19 #Alexnet3.promise_accuracy = 90.19 Alexnet3.promise_accuracy = 89.55 @@ -164,7 +172,7 @@ Alexnet4.layer_knobs = "../opentuner/data/resnet/knobs.txt" Alexnet4.loss1_result_file = "tuner_results/resnet18_cifar10/loss_1/promise_tuned_confs/promise_confs.txt" Alexnet4.loss2_result_file = "tuner_results/resnet18_cifar10/loss_2/promise_tuned_confs/promise_confs.txt" -Alexnet4.autotuner_runs = 1500 +Alexnet4.autotuner_runs = 500 Alexnet4.tuner_accuracy = 89.6 #Alexnet4.promise_accuracy = 89.59 - 1000 images Alexnet4.promise_accuracy = 89.94 @@ -199,7 +207,7 @@ Alexnet5.layer_knobs = "../opentuner/data/vgg16_cifar100/knobs.txt" Alexnet5.loss1_result_file = "tuner_results/vgg_cifar100/loss_1/promise_tuned_confs/promise_confs.txt" Alexnet5.loss2_result_file = "tuner_results/vgg_cifar100/loss_2/promise_tuned_confs/promise_confs.txt" -Alexnet5.autotuner_runs = 1500 +Alexnet5.autotuner_runs = 500 Alexnet5.tuner_accuracy = 67.95 #Alexnet5.promise_accuracy = 66.8 Alexnet5.promise_accuracy = 70.1 @@ -233,7 +241,7 @@ Alexnet6.layer_knobs = "../opentuner/data/lenet/knobs.txt" #Alexnet6.loss1_result_file = "tuner_results/vgg_cifar100/loss_1/promise_tuned_confs/promise_confs.txt" #Alexnet6.loss2_result_file = "tuner_results/vgg_cifar100/loss_2/promise_tuned_confs/promise_confs.txt" -Alexnet6.autotuner_runs = 900 +Alexnet6.autotuner_runs = 500 Alexnet6.tuner_accuracy = 98.9 Alexnet6.promise_accuracy = 98.9 Alexnet6.validation_accuracy = 99 @@ -268,7 +276,7 @@ Alexnet7.layer_knobs = "../opentuner/data/mobilenet/knobs.txt" #--- Files below needed for VALIDATION experiment Alexnet7.loss1_result_file = "tuner_results/mobilenet/loss_1/batch1/promise_tuner/high_confidence/promise_confs.txt" Alexnet7.loss2_result_file = "tuner_results/mobilenet/loss_2/batch1/promise_tuner/high_confidence/promise_confs.txt" -Alexnet7.autotuner_runs = 1500 +Alexnet7.autotuner_runs = 500 Alexnet7.tuner_accuracy = 84.8 #Alexnet7.promise_accuracy = 84.8 Alexnet7.promise_accuracy = 83.65 @@ -302,7 +310,7 @@ Alexnet8.layer_knobs = "../opentuner/data/mobilenet_shallow/knobs.txt" Alexnet8.loss1_result_file = "../build_tuner/tuner_results/mobilenet_shallow/loss_1/batch2/promise_tuner/high_confidence/promise_selected_confs.txt" Alexnet8.loss2_result_file = "../build_tuner/tuner_results/mobilenet_shallow/loss_2/batch2/promise_tuner/high_confidence/promise_selected_confs.txt" -Alexnet8.autotuner_runs = 1500 +Alexnet8.autotuner_runs = 500 Alexnet8.tuner_accuracy = 87.6 #Alexnet8.promise_accuracy = 87.59 Alexnet8.promise_accuracy = 89.25 diff --git a/llvm/projects/hpvm-tensor-rt/bin/tuner_src/buildRtConfig.py b/llvm/projects/hpvm-tensor-rt/bin/tuner_src/buildRtConfig.py index 6a07ef86e53d2b4b6372e1e253611ba6f018aaad..f82b1a24f3982802c249f502161d944110b389e0 100644 --- a/llvm/projects/hpvm-tensor-rt/bin/tuner_src/buildRtConfig.py +++ b/llvm/projects/hpvm-tensor-rt/bin/tuner_src/buildRtConfig.py @@ -89,6 +89,9 @@ def skipFile(fname): skip_files["confidence_summary.txt"] = 1 skip_files["promise_confs.txt"] = 1 + if "accuracy" in fname: # *_accuracy files should be skipped + return True + if fname in skip_files: return True else: @@ -117,7 +120,8 @@ def loadConfigData(result_dir, baseline_accuracy): config_arr = [] - result_dir += "/promise_tuner/high_confidence/" + #result_dir += "/promise_tuner/high_confidence/" + result_dir += "/algo_tuner/high_confidence/" file_names = os.listdir(result_dir) diff --git a/llvm/projects/hpvm-tensor-rt/bin/tuner_src/run_autotuner.py b/llvm/projects/hpvm-tensor-rt/bin/tuner_src/run_autotuner.py index 73d460be0c4091067c9d52e07ea7f4d421765ff3..1e533b8702139966166f860b72a3df1ccae03ee6 100644 --- a/llvm/projects/hpvm-tensor-rt/bin/tuner_src/run_autotuner.py +++ b/llvm/projects/hpvm-tensor-rt/bin/tuner_src/run_autotuner.py @@ -12,7 +12,7 @@ from run_ha_tuner import runTunerBench from run_hs_tuner import runPromiseBench from run_algo_tuner import runAlgoBench from compute_confs import computePSNRBenchSwings, computeBenchSwings -from validation import runPromiseBenchValidation, runPromiseBenchValidation2, runBenchValidation +from validation import runPromiseBenchValidation2, runBenchValidation, runAlgoBenchValidate from profiling import startProfile, stopProfile, dumpProfiles from utils import createResultDirs from benchmarks import batch_id @@ -276,8 +276,8 @@ def runAlgoTuner(): Bench = bench_tuner_data["vgg16_cifar10"] runAlgoBench(Bench) - #Bench = bench_tuner_data["lenet_keras"] - #runAlgoBench(Bench) + Bench = bench_tuner_data["lenet_keras"] + runAlgoBench(Bench) Bench = bench_tuner_data["alexnet2_cifar10"] runAlgoBench(Bench) @@ -292,6 +292,36 @@ def runAlgoTuner(): +def runAlgoTunerValidation(): + + + Bench = bench_tuner_data["alexnet_cifar10"] + runAlgoBenchValidate(Bench) + + Bench = bench_tuner_data["mobilenet_shallow"] + runAlgoBenchValidate(Bench) + + Bench = bench_tuner_data["mobilenet_cifar10"] + runAlgoBenchValidate(Bench) + + Bench = bench_tuner_data["vgg16_cifar10"] + runAlgoBenchValidate(Bench) + + Bench = bench_tuner_data["lenet_keras"] + runAlgoBenchValidate(Bench) + + Bench = bench_tuner_data["alexnet2_cifar10"] + runAlgoBenchValidate(Bench) + + Bench = bench_tuner_data["vgg16_cifar100"] + runAlgoBenchValidate(Bench) + + Bench = bench_tuner_data["resnet18_cifar10"] + runAlgoBenchValidate(Bench) + + + + if __name__ == "__main__": @@ -305,7 +335,9 @@ if __name__ == "__main__": #runPromiseTuner() - runAlgoTuner() + #runAlgoTuner() + + runAlgoTunerValidation() #runPromiseValidation() diff --git a/llvm/projects/hpvm-tensor-rt/bin/tuner_src/validation.py b/llvm/projects/hpvm-tensor-rt/bin/tuner_src/validation.py index 586d23d70a661558bad0cb8ac75c2367e3e73e2f..06a076103b8052d7321951f39bab738ec6d58358 100644 --- a/llvm/projects/hpvm-tensor-rt/bin/tuner_src/validation.py +++ b/llvm/projects/hpvm-tensor-rt/bin/tuner_src/validation.py @@ -488,7 +488,7 @@ def runPromiseBenchValidation(Bench): def copyValidatedConf(result_dir, validated_confs): src_dir = result_dir + "/promise_tuner/high_confidence/" - dest_dir = result_dir + "/promise_tuner/validated/" + dest_dir = result_dir + "/promise_tuner/validated_test/" if not os.path.isdir(dest_dir): os.mkdir(dest_dir) @@ -501,7 +501,7 @@ def copyValidatedConf(result_dir, validated_confs): def copyFailedConf(result_dir, failed_confs): src_dir = result_dir + "/promise_tuner/high_confidence/" - dest_dir = result_dir + "/promise_tuner/failed/" + dest_dir = result_dir + "/promise_tuner/failed_test/" if not os.path.isdir(dest_dir): os.mkdir(dest_dir) @@ -529,6 +529,9 @@ def validateConfigs(Bench, result_dir, configs_arr, acc_thresh): copyFailedConf(result_dir, failed_confs) + + + def runPromiseBenchValidation2(Bench): @@ -543,3 +546,94 @@ def runPromiseBenchValidation2(Bench): validateConfigs(Bench, Bench.result_dir_2, config_arr2, 2.0) validateConfigs(Bench, Bench.result_dir_3, config_arr3, 3.0) + + + +### NOTE: Algo Tuner Validation routines + + +def readConfidence(target_acc): + + f = open("run_accuracies.txt") + index = 0.0 + unsuccessful = 0.0 + sum_acc = 0.0 + for x in f: + x = x.strip() + acc = float(x) + if acc < target_acc: + unsuccessful += 1 + index += 1 + sum_acc += acc + + f.close() + + confidence = ( (index - unsuccessful) / index) * 100.0 + print ("run_confidence = ", confidence) + avg_acc = sum_acc / index + + return confidence + + + +def invokeBinary(Bench, layer_swings, target_acc, runs): # threshold): + + validation_binary = Bench.promise_binary + + # Write to promise_flags + fout = open("promise_flags", "w+") + for swing in layer_swings: + int_swing = int(swing) + if int_swing > 0: + fout.write(str(swing) + "\n") + fout.close() + + # Execute Validation Run + p = subprocess.Popen("./" + validation_binary, shell=True) + p.wait() + + + + + +def validateAlgoConfigs(Bench, result_dir, configs_arr, acc_thresh): + + # NOTE: Use confidence target as 95% + confidence_target = 95 + # NOTE: 1 run sufficient for software approximations + runs = 1 + + validated_confs = [] + failed_confs = [] + + validation_acc = Bench.validation_accuracy + target_acc = validation_acc - acc_thresh + + for conf in configs_arr: + layer_swings = conf.flags + invokeBinary(Bench, layer_swings, target_acc, runs) # acc_thresh) + confidence = readConfidence(target_acc) + + if confidence >= confidence_target: + validated_confs.append(conf.fname) + else: + failed_confs.append(conf.fname) + + + copyValidatedConf(result_dir, validated_confs) + copyFailedConf(result_dir, failed_confs) + + + + +def runAlgoBenchValidate(Bench): + + config_arr1 = loadConfigData(Bench.result_dir_1, 100) + config_arr2 = loadConfigData(Bench.result_dir_2, 100) + config_arr3 = loadConfigData(Bench.result_dir_3, 100) + + + validateAlgoConfigs(Bench, Bench.result_dir_1, config_arr1, 1.0) + validateAlgoConfigs(Bench, Bench.result_dir_2, config_arr2, 2.0) + validateAlgoConfigs(Bench, Bench.result_dir_3, config_arr3, 3.0) +