diff --git a/llvm/projects/hpvm-tensor-rt/bin/run_autotuner.py b/llvm/projects/hpvm-tensor-rt/bin/run_autotuner.py index 58080191bdb27b311d6422ddb50bff7f92b234a0..e08d007ec40bdf51ee0ef2485a21fbdd1800cb79 100644 --- a/llvm/projects/hpvm-tensor-rt/bin/run_autotuner.py +++ b/llvm/projects/hpvm-tensor-rt/bin/run_autotuner.py @@ -35,10 +35,12 @@ Alexnet1.num_flags = 21 Alexnet1.num_layers = 6 Alexnet1.error_range_1 = 9 Alexnet1.error_range_2 = 12 +Alexnet1.start_promise_range = 4 Alexnet1.result_dir_1 = "tuner_results/alexnet_cifar10/loss_1/batch1" Alexnet1.result_dir_2 = "tuner_results/alexnet_cifar10/loss_2/batch1" Alexnet1.tensor_desc_file = "tuner_results/alexnet_cifar10/alexnet_tensors.txt" Alexnet1.layer_file = "tuner_results/alexnet_cifar10/alexnet_layers.txt" +#Alexnet1.loss1_result_file = "tuner_results/alexnet2_cifar10/alexnet_layers.txt" Alexnet1.autotuner_runs = 1000 Alexnet1.tuner_accuracy = 78.9 Alexnet1.promise_accuracy = 78.9 @@ -49,16 +51,20 @@ bench_tuner_data["alexnet_cifar10"] = Alexnet1 Alexnet2 = Benchmark() Alexnet2.tuner_binary = "alexnet2_cifar10_tuner" -Alexnet2.promise_binary = "alexnet2_promise" +Alexnet2.promise_binary = "alexnet2_promise_quant" Alexnet2.validation_binary = "alexnet2_valid" Alexnet2.num_flags = 23 Alexnet2.num_layers = 7 Alexnet2.error_range_1 = 9 Alexnet2.error_range_2 = 11 -Alexnet2.result_dir_1 = "tuner_results/alexnet2_cifar10/loss_1/batch2" -Alexnet2.result_dir_2 = "tuner_results/alexnet2_cifar10/loss_2/batch2" +Alexnet2.start_promise_range = 4 +Alexnet2.result_dir_1 = "tuner_results/alexnet2_cifar10/loss_1/batch1" +Alexnet2.result_dir_2 = "tuner_results/alexnet2_cifar10/loss_2/batch1" Alexnet2.tensor_desc_file = "tuner_results/alexnet2_cifar10/alexnet2_tensors.txt" Alexnet2.layer_file = "tuner_results/alexnet2_cifar10/alexnet2_layers.txt" + +Alexnet2.loss1_result_file = "tuner_results/alexnet2_cifar10/loss_1/promise_tuned_confs/promise_confs.txt" +Alexnet2.loss2_result_file = "tuner_results/alexnet2_cifar10/loss_2/promise_tuned_confs/promise_confs.txt" Alexnet2.autotuner_runs = 800 Alexnet2.tuner_accuracy = 84.63 Alexnet2.promise_accuracy = 84.63 @@ -95,13 +101,17 @@ Alexnet4.tuner_binary = "resnet18_cifar10_tuner" Alexnet4.promise_binary = "resnet18_promise_quant" Alexnet4.validation_binary = "resnet18_valid" Alexnet4.num_flags = 73 -Alexnet4.num_layers = 22 +Alexnet4.num_layers = 23 Alexnet4.error_range_1 = 7 Alexnet4.error_range_2 = 9 -Alexnet4.result_dir_1 = "tuner_results/resnet18_cifar10/loss_1/batch1" -Alexnet4.result_dir_2 = "tuner_results/resnet18_cifar10/loss_2/batch1" +Alexnet4.start_promise_range = 7 +Alexnet4.result_dir_1 = "tuner_results/resnet18_cifar10/loss_1/batch3" +Alexnet4.result_dir_2 = "tuner_results/resnet18_cifar10/loss_2/batch3" Alexnet4.tensor_desc_file = "tuner_results/resnet18_cifar10/resnet_tensors.txt" Alexnet4.layer_file = "tuner_results/resnet18_cifar10/resnet_layers.txt" + +Alexnet4.loss1_result_file = "tuner_results/resnet18_cifar10/loss_1/promise_tuned_confs/promise_confs.txt" +Alexnet4.loss2_result_file = "tuner_results/resnet18_cifar10/loss_2/promise_tuned_confs/promise_confs.txt" Alexnet4.autotuner_runs = 1500 Alexnet4.tuner_accuracy = 89.54 Alexnet4.promise_accuracy = 89.54 @@ -113,6 +123,32 @@ bench_tuner_data["resnet18_cifar10"] = Alexnet4 +Alexnet5 = Benchmark() +Alexnet5.tuner_binary = "vgg16_cifar100_tuner" +Alexnet5.promise_binary = "vgg16_cifar100_promise_quant" +Alexnet5.validation_binary = "vgg16_cifar100_valid" +Alexnet5.num_flags = 50 +Alexnet5.num_layers = 15 +Alexnet5.error_range_1 = 9 +Alexnet5.error_range_2 = 11 +Alexnet5.start_promise_range = 5 +Alexnet5.result_dir_1 = "tuner_results/vgg16_cifar100/loss_1/batch2" +Alexnet5.result_dir_2 = "tuner_results/vgg16_cifar100/loss_2/batch2" +Alexnet5.tensor_desc_file = "tuner_results/vgg16_cifar100/vgg16_tensors.txt" +Alexnet5.layer_file = "tuner_results/vgg16_cifar100/vgg16_layers.txt" + +Alexnet5.loss1_result_file = "tuner_results/vgg_cifar100/loss_1/promise_tuned_confs/promise_confs.txt" +Alexnet5.loss2_result_file = "tuner_results/vgg_cifar100/loss_2/promise_tuned_confs/promise_confs.txt" +Alexnet5.autotuner_runs = 1000 +Alexnet5.tuner_accuracy = 69.09 +Alexnet5.promise_accuracy = 69.09 +Alexnet5.validation_accuracy = 66.15 + +bench_tuner_data["vgg16_cifar100"] = Alexnet5 + + + + def runTunerBench(bench_name): @@ -261,7 +297,8 @@ def getLayerString(layer_swings): def testValidationRun(Bench, validation_dir, layer_swings, threshold, rank_str): - os.chdir("../build_promise/") + #### FIXME + #os.chdir("../build_promise/") validation_acc = Bench.validation_accuracy target_acc = validation_acc - threshold @@ -271,7 +308,9 @@ def testValidationRun(Bench, validation_dir, layer_swings, threshold, rank_str): # Write to promise_flags fout = open("promise_flags", "w+") for swing in layer_swings: - fout.write(str(swing) + "\n") + int_swing = int(swing) + if int_swing > 0: + fout.write(str(swing) + "\n") fout.close() # Execute Validation Run @@ -314,10 +353,57 @@ def testValidationRun(Bench, validation_dir, layer_swings, threshold, rank_str): +def testPromiseRun(Bench, layer_swings, threshold): + + #### FIXME + #os.chdir("../build_promise/") + + validation_acc = Bench.validation_accuracy + target_acc = validation_acc - threshold + + validation_binary = Bench.validation_binary + + # Write to promise_flags + fout = open("promise_flags", "w+") + for swing in layer_swings: + int_swing = int(swing) + if int_swing > 0: + fout.write(str(swing) + "\n") + fout.close() + + # Execute Validation Run + p = subprocess.Popen("./" + validation_binary, shell=True) + p.wait() + + f = open("run_accuracies.txt") + index = 0.0 + unsuccessful = 0.0 + sum_acc = 0.0 + for x in f: + x = x.strip() + acc = float(x) + if acc < target_acc: + unsuccessful += 1 + index += 1 + sum_acc += acc + + f.close() + + confidence = ( (index - unsuccessful) / index) * 100.0 + print ("run_confidence = ", confidence) + avg_acc = sum_acc / index + + return confidence + + + + + def dumpConfigConfidence(configs, confidence_list, result_dir, layer_desc_file): - os.chdir("../build_tuner/") + #### FIXME + #os.chdir("../build_tuner/") layer_desc = loadLayerDesc(layer_desc_file) print (layer_desc) @@ -477,13 +563,90 @@ def runBenchValidation(bench_name): print (validated_confs1) print (validated_confs2) + + +def readPromiseResults(loss1_file, loss2_file): + + loss_confs = [] + loss1_confs = [] + f1 = open(loss1_file) + for x in f1: + print (x) + swing_toks = x.split(",") + swing_list = [] + for swing_str in swing_toks: + swing_val = int(swing_str.split(" ")[0]) + swing_list.append(swing_val) + loss1_confs.append(swing_list) + + loss_confs.append(loss1_confs) + + loss2_confs = [] + f2 = open(loss1_file) + for x in f2: + swing_toks = x.split(",") + swing_list = [] + for swing_str in swing_toks: + swing_val = int(swing_str.split(" ")[0]) + swing_list.append(swing_val) + loss2_confs.append(swing_list) + + loss_confs.append(loss2_confs) + + return loss_confs + + +def runPromiseBenchValidation(bench_name): + + Bench = bench_tuner_data[bench_name] + loss_confs = readPromiseResults(Bench.loss1_result_file, Bench.loss2_result_file) + loss1_confs = loss_confs[0] + loss2_confs = loss_confs[1] + + ind = 0 + validated_confs1 = [] + failed_confs1 = [] + for layer_swings in loss1_confs: + confidence = testPromiseRun(Bench, layer_swings, 1.0) + if confidence > 95: + validated_confs1.append(layer_swings) + else: + failed_confs1.append(layer_swings) + ind += 1 + + + ind = 0 + validated_confs2 = [] + failed_confs2 = [] + for layer_swings in loss2_confs: + confidence = testPromiseRun(Bench, layer_swings, 2.0) + if confidence > 95: + validated_confs2.append(layer_swings) + else: + failed_confs2.append(layer_swings) + ind += 1 + + dumpValidatedConfigs(validated_confs1, Bench.result_dir_1, + Bench.layer_file, "promise_validated_confs.txt") + dumpValidatedConfigs(validated_confs2, Bench.result_dir_2, + Bench.layer_file, "promise_validated_confs.txt") + + dumpValidatedConfigs(failed_confs1, Bench.result_dir_1, + Bench.layer_file, "promise_failed_confs.txt") + dumpValidatedConfigs(failed_confs2, Bench.result_dir_2, + Bench.layer_file, "promise_failed_confs.txt") + + + def runTunerValidation(): - runBenchValidation("alexnet_cifar10") - runBenchValidation("vgg16_cifar10") + #runBenchValidation("alexnet_cifar10") + #runBenchValidation("vgg16_cifar10") #runBenchValidation("alexnet2_cifar10") + #runBenchValidation("resnet18_cifar10") + runBenchValidation("vgg16_cifar100") def computeLayerSwings(): @@ -501,9 +664,13 @@ def runPromiseTuner(): def runPromiseValidation(): - return + + #runPromiseBenchValidation("resnet18_cifar10") + runPromiseBenchValidation("alexnet2_cifar10") + #runPromiseBenchValidation("vgg_cifar100") + def runAutotuner(): #runTunerBench("resnet18_cifar10") @@ -516,10 +683,10 @@ if __name__ == "__main__": #runAutotuner() - runTunerValidation() + #runTunerValidation() #computeLayerSwings() #runPromiseTuner() - #runPromiseValidation() + runPromiseValidation()