Skip to content
Snippets Groups Projects
Commit 7b661285 authored by Ubuntu's avatar Ubuntu
Browse files

Adding Promise Validation

parent 4be13187
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
llvm/projects/hpvm-tensor-rt/bin/run_autotuner.py
+ 181
− 14
View file @ 7b661285
......@@ -35,10 +35,12 @@ Alexnet1.num_flags = 21
Alexnet1.num_layers = 6
Alexnet1.error_range_1 = 9
Alexnet1.error_range_2 = 12
Alexnet1.start_promise_range = 4
Alexnet1.result_dir_1 = "tuner_results/alexnet_cifar10/loss_1/batch1"
Alexnet1.result_dir_2 = "tuner_results/alexnet_cifar10/loss_2/batch1"
Alexnet1.tensor_desc_file = "tuner_results/alexnet_cifar10/alexnet_tensors.txt"
Alexnet1.layer_file = "tuner_results/alexnet_cifar10/alexnet_layers.txt"
#Alexnet1.loss1_result_file = "tuner_results/alexnet2_cifar10/alexnet_layers.txt"
Alexnet1.autotuner_runs = 1000
Alexnet1.tuner_accuracy = 78.9
Alexnet1.promise_accuracy = 78.9
......@@ -49,16 +51,20 @@ bench_tuner_data["alexnet_cifar10"] = Alexnet1
Alexnet2 = Benchmark()
Alexnet2.tuner_binary = "alexnet2_cifar10_tuner"
Alexnet2.promise_binary = "alexnet2_promise"
Alexnet2.promise_binary = "alexnet2_promise_quant"
Alexnet2.validation_binary = "alexnet2_valid"
Alexnet2.num_flags = 23
Alexnet2.num_layers = 7
Alexnet2.error_range_1 = 9
Alexnet2.error_range_2 = 11
Alexnet2.result_dir_1 = "tuner_results/alexnet2_cifar10/loss_1/batch2"
Alexnet2.result_dir_2 = "tuner_results/alexnet2_cifar10/loss_2/batch2"
Alexnet2.start_promise_range = 4
Alexnet2.result_dir_1 = "tuner_results/alexnet2_cifar10/loss_1/batch1"
Alexnet2.result_dir_2 = "tuner_results/alexnet2_cifar10/loss_2/batch1"
Alexnet2.tensor_desc_file = "tuner_results/alexnet2_cifar10/alexnet2_tensors.txt"
Alexnet2.layer_file = "tuner_results/alexnet2_cifar10/alexnet2_layers.txt"
Alexnet2.loss1_result_file = "tuner_results/alexnet2_cifar10/loss_1/promise_tuned_confs/promise_confs.txt"
Alexnet2.loss2_result_file = "tuner_results/alexnet2_cifar10/loss_2/promise_tuned_confs/promise_confs.txt"
Alexnet2.autotuner_runs = 800
Alexnet2.tuner_accuracy = 84.63
Alexnet2.promise_accuracy = 84.63
......@@ -95,13 +101,17 @@ Alexnet4.tuner_binary = "resnet18_cifar10_tuner"
Alexnet4.promise_binary = "resnet18_promise_quant"
Alexnet4.validation_binary = "resnet18_valid"
Alexnet4.num_flags = 73
Alexnet4.num_layers = 22
Alexnet4.num_layers = 23
Alexnet4.error_range_1 = 7
Alexnet4.error_range_2 = 9
Alexnet4.result_dir_1 = "tuner_results/resnet18_cifar10/loss_1/batch1"
Alexnet4.result_dir_2 = "tuner_results/resnet18_cifar10/loss_2/batch1"
Alexnet4.start_promise_range = 7
Alexnet4.result_dir_1 = "tuner_results/resnet18_cifar10/loss_1/batch3"
Alexnet4.result_dir_2 = "tuner_results/resnet18_cifar10/loss_2/batch3"
Alexnet4.tensor_desc_file = "tuner_results/resnet18_cifar10/resnet_tensors.txt"
Alexnet4.layer_file = "tuner_results/resnet18_cifar10/resnet_layers.txt"
Alexnet4.loss1_result_file = "tuner_results/resnet18_cifar10/loss_1/promise_tuned_confs/promise_confs.txt"
Alexnet4.loss2_result_file = "tuner_results/resnet18_cifar10/loss_2/promise_tuned_confs/promise_confs.txt"
Alexnet4.autotuner_runs = 1500
Alexnet4.tuner_accuracy = 89.54
Alexnet4.promise_accuracy = 89.54
......@@ -113,6 +123,32 @@ bench_tuner_data["resnet18_cifar10"] = Alexnet4
Alexnet5 = Benchmark()
Alexnet5.tuner_binary = "vgg16_cifar100_tuner"
Alexnet5.promise_binary = "vgg16_cifar100_promise_quant"
Alexnet5.validation_binary = "vgg16_cifar100_valid"
Alexnet5.num_flags = 50
Alexnet5.num_layers = 15
Alexnet5.error_range_1 = 9
Alexnet5.error_range_2 = 11
Alexnet5.start_promise_range = 5
Alexnet5.result_dir_1 = "tuner_results/vgg16_cifar100/loss_1/batch2"
Alexnet5.result_dir_2 = "tuner_results/vgg16_cifar100/loss_2/batch2"
Alexnet5.tensor_desc_file = "tuner_results/vgg16_cifar100/vgg16_tensors.txt"
Alexnet5.layer_file = "tuner_results/vgg16_cifar100/vgg16_layers.txt"
Alexnet5.loss1_result_file = "tuner_results/vgg_cifar100/loss_1/promise_tuned_confs/promise_confs.txt"
Alexnet5.loss2_result_file = "tuner_results/vgg_cifar100/loss_2/promise_tuned_confs/promise_confs.txt"
Alexnet5.autotuner_runs = 1000
Alexnet5.tuner_accuracy = 69.09
Alexnet5.promise_accuracy = 69.09
Alexnet5.validation_accuracy = 66.15
bench_tuner_data["vgg16_cifar100"] = Alexnet5
def runTunerBench(bench_name):
......@@ -261,7 +297,8 @@ def getLayerString(layer_swings):
def testValidationRun(Bench, validation_dir, layer_swings, threshold, rank_str):
os.chdir("../build_promise/")
#### FIXME
#os.chdir("../build_promise/")
validation_acc = Bench.validation_accuracy
target_acc = validation_acc - threshold
......@@ -271,7 +308,9 @@ def testValidationRun(Bench, validation_dir, layer_swings, threshold, rank_str):
# Write to promise_flags
fout = open("promise_flags", "w+")
for swing in layer_swings:
fout.write(str(swing) + "\n")
int_swing = int(swing)
if int_swing > 0:
fout.write(str(swing) + "\n")
fout.close()
# Execute Validation Run
......@@ -314,10 +353,57 @@ def testValidationRun(Bench, validation_dir, layer_swings, threshold, rank_str):
def testPromiseRun(Bench, layer_swings, threshold):
#### FIXME
#os.chdir("../build_promise/")
validation_acc = Bench.validation_accuracy
target_acc = validation_acc - threshold
validation_binary = Bench.validation_binary
# Write to promise_flags
fout = open("promise_flags", "w+")
for swing in layer_swings:
int_swing = int(swing)
if int_swing > 0:
fout.write(str(swing) + "\n")
fout.close()
# Execute Validation Run
p = subprocess.Popen("./" + validation_binary, shell=True)
p.wait()
f = open("run_accuracies.txt")
index = 0.0
unsuccessful = 0.0
sum_acc = 0.0
for x in f:
x = x.strip()
acc = float(x)
if acc < target_acc:
unsuccessful += 1
index += 1
sum_acc += acc
f.close()
confidence = ( (index - unsuccessful) / index) * 100.0
print ("run_confidence = ", confidence)
avg_acc = sum_acc / index
return confidence
def dumpConfigConfidence(configs, confidence_list,
result_dir, layer_desc_file):
os.chdir("../build_tuner/")
#### FIXME
#os.chdir("../build_tuner/")
layer_desc = loadLayerDesc(layer_desc_file)
print (layer_desc)
......@@ -477,13 +563,90 @@ def runBenchValidation(bench_name):
print (validated_confs1)
print (validated_confs2)
def readPromiseResults(loss1_file, loss2_file):
loss_confs = []
loss1_confs = []
f1 = open(loss1_file)
for x in f1:
print (x)
swing_toks = x.split(",")
swing_list = []
for swing_str in swing_toks:
swing_val = int(swing_str.split(" ")[0])
swing_list.append(swing_val)
loss1_confs.append(swing_list)
loss_confs.append(loss1_confs)
loss2_confs = []
f2 = open(loss1_file)
for x in f2:
swing_toks = x.split(",")
swing_list = []
for swing_str in swing_toks:
swing_val = int(swing_str.split(" ")[0])
swing_list.append(swing_val)
loss2_confs.append(swing_list)
loss_confs.append(loss2_confs)
return loss_confs
def runPromiseBenchValidation(bench_name):
Bench = bench_tuner_data[bench_name]
loss_confs = readPromiseResults(Bench.loss1_result_file, Bench.loss2_result_file)
loss1_confs = loss_confs[0]
loss2_confs = loss_confs[1]
ind = 0
validated_confs1 = []
failed_confs1 = []
for layer_swings in loss1_confs:
confidence = testPromiseRun(Bench, layer_swings, 1.0)
if confidence > 95:
validated_confs1.append(layer_swings)
else:
failed_confs1.append(layer_swings)
ind += 1
ind = 0
validated_confs2 = []
failed_confs2 = []
for layer_swings in loss2_confs:
confidence = testPromiseRun(Bench, layer_swings, 2.0)
if confidence > 95:
validated_confs2.append(layer_swings)
else:
failed_confs2.append(layer_swings)
ind += 1
dumpValidatedConfigs(validated_confs1, Bench.result_dir_1,
Bench.layer_file, "promise_validated_confs.txt")
dumpValidatedConfigs(validated_confs2, Bench.result_dir_2,
Bench.layer_file, "promise_validated_confs.txt")
dumpValidatedConfigs(failed_confs1, Bench.result_dir_1,
Bench.layer_file, "promise_failed_confs.txt")
dumpValidatedConfigs(failed_confs2, Bench.result_dir_2,
Bench.layer_file, "promise_failed_confs.txt")
def runTunerValidation():
runBenchValidation("alexnet_cifar10")
runBenchValidation("vgg16_cifar10")
#runBenchValidation("alexnet_cifar10")
#runBenchValidation("vgg16_cifar10")
#runBenchValidation("alexnet2_cifar10")
#runBenchValidation("resnet18_cifar10")
runBenchValidation("vgg16_cifar100")
def computeLayerSwings():
......@@ -501,9 +664,13 @@ def runPromiseTuner():
def runPromiseValidation():
return
#runPromiseBenchValidation("resnet18_cifar10")
runPromiseBenchValidation("alexnet2_cifar10")
#runPromiseBenchValidation("vgg_cifar100")
def runAutotuner():
#runTunerBench("resnet18_cifar10")
......@@ -516,10 +683,10 @@ if __name__ == "__main__":
#runAutotuner()
runTunerValidation()
#runTunerValidation()
#computeLayerSwings()
#runPromiseTuner()
#runPromiseValidation()
runPromiseValidation()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment