From 387e09efe9c7026bc13a537ed62de4ff10c7fbb3 Mon Sep 17 00:00:00 2001
From: Hashim Sharif <hsharif3@tyler.cs.illinois.edu>
Date: Fri, 11 Oct 2019 05:16:53 -0500
Subject: [PATCH] Incorporating Pareto Curve for Conf Measurement
---
.../autotuner/measure_confidence2.py | 58 +++-
.../opentuner/autotuner/pareto_curve.py | 259 ++++++++++++++++++
.../opentuner/autotuner/promise_tuner3.py | 21 +-
3 files changed, 328 insertions(+), 10 deletions(-)
create mode 100644 llvm/projects/hpvm-tensor-rt/opentuner/autotuner/pareto_curve.py
diff --git a/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/measure_confidence2.py b/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/measure_confidence2.py
index 80e67a1bc3..b38efa9c82 100644
--- a/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/measure_confidence2.py
+++ b/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/measure_confidence2.py
@@ -19,8 +19,8 @@ def getAccuracy(file_name):
return accuracy
-total_runs = 60.0
-fails_allowed = 4
+total_runs = 40.0
+fails_allowed = 3
skip_lines = 0
@@ -297,6 +297,8 @@ def getConfigCost(layer_costs, config_str):
continue
orig_cost += layer_costs[it]
+
+ #print ("orig_cost = ", orig_cost, " flag_value = ", flag_value)
if flag_value == 11:
total_cost += layer_costs[it]
@@ -307,10 +309,12 @@ def getConfigCost(layer_costs, config_str):
elif flag_value < 8:
divisor = 5 + (7 - flag_value)
total_cost += (layer_costs[it] / divisor)
-
+
it += 1
-
- return total_cost, (orig_cost / total_cost)
+
+ speedup = orig_cost * 1.0 / total_cost * 1.0
+
+ return total_cost, speedup
@@ -406,6 +410,7 @@ def dump_promise_confidence_files(binary, result_dir, layer_file_path,
def dump_promise_confidence_files2(binary, result_dir, layer_file_path,
num_flags, accuracy, layer_costs, confidence):
+
#result_dir = args.result_dir
output_dir = result_dir + "/high_confidence"
input_dir = result_dir + "/full_results"
@@ -445,6 +450,49 @@ def dump_promise_confidence_files2(binary, result_dir, layer_file_path,
+
+def dump_promise_confidence_files3(binary, input_dir, output_dir, layer_file_path,
+ num_flags, accuracy, layer_costs, confidence):
+
+
+ #result_dir = args.result_dir
+ #output_dir = result_dir + "/high_confidence"
+ #input_dir = result_dir + "/full_results"
+
+ if not os.path.exists(output_dir):
+ os.mkdir(output_dir)
+
+ layer_sizes = processLayerFile(layer_file_path);
+ print layer_sizes
+ sleep(2)
+
+ confidence_list = compute_promise_confidence2(binary, accuracy, confidence, layer_costs, input_dir, output_dir)
+ print confidence_list
+
+ # Ascending sort on accuracy
+ sorted_list = sorted(confidence_list, key = lambda tup: tup[1])
+
+ promise_file = open(output_dir + "/promise_confs.txt", "w+")
+ confidence_file = open(output_dir + "/confidence_summary.txt", "w+")
+
+ max_configs = 50
+ it_count = 0
+ for x in sorted_list:
+ if x[1] > accuracy and x[0] > confidence:
+ config_str = getLayerConfigStr(x[3], layer_sizes, num_flags)
+ promise_file.write(config_str + "\n")
+ it_count += 1
+ if it_count > max_configs:
+ break
+
+ confidence_file.write(str(x[0]) + "\t" + str(x[1]) + "\t" + str(x[3]) + "\n")
+
+ promise_file.close()
+ confidence_file.close()
+
+ print "Dumped Confidence Summary"
+
+
diff --git a/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/pareto_curve.py b/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/pareto_curve.py
new file mode 100644
index 0000000000..0fda8f742c
--- /dev/null
+++ b/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/pareto_curve.py
@@ -0,0 +1,259 @@
+
+
+import os
+import shutil
+from measure_confidence2 import getConfigCost
+
+
+class Config:
+ def __init__(self):
+ self.avg_accuracy = 0
+ self.avg_loss = 0
+ self.speedup = 1
+ self.fname = ""
+ self.flags = []
+
+
+
+
+def skipFile(fname):
+
+ skip_files = {}
+ skip_files["confidence_summary.txt"] = 1
+ skip_files["promise_confs.txt"] = 1
+
+ if "accuracy" in fname:
+ return True
+
+ if fname in skip_files:
+ return True
+ else:
+ return False
+
+
+
+
+def loadConfigData(result_dir, layer_costs, baseline_accuracy):
+
+ config_arr = []
+
+ #result_dir += "/promise_tuner/high_confidence/"
+ file_names = os.listdir(result_dir)
+
+
+ for fname in file_names:
+ if not skipFile(fname):
+
+ fpath = result_dir + fname
+ config = Config()
+ f = open(fpath, "r")
+
+ config_str = f.read()
+ cost, speedup = getConfigCost(layer_costs, config_str)
+
+ config.speedup = speedup
+ config.fname = fname
+
+ fpath2 = fpath + "_accuracy"
+ f2 = open(fpath2, "r")
+ acc_str = f2.read().strip()
+ accuracy = float(acc_str)
+
+ config.avg_accuracy = accuracy
+ config.avg_loss = baseline_accuracy - accuracy
+
+ config_arr.append(config)
+
+
+ return config_arr
+
+
+
+AL_THRESHOLD = 0.1
+SPEEDUP_BAND_SIZE = 0.3
+ENERGY_BAND_SIZE = 10
+
+
+class Configuration:
+ def __init__(self, name, speedup, energy, accuracy, accuracy_loss):
+ self.name = name
+ self.speedup = speedup
+ self.energy = energy
+ self.accuracy = accuracy
+ self.accuracy_loss = accuracy_loss
+ def __repr__(self):
+ return repr((self.name, self.speedup, self.energy, self.accuracy, self.accuracy_loss))
+
+configuration_objects = [
+ Configuration('conf1', 1.05, 15, 85, 1.2),
+ Configuration('conf2', 2.51, 12, 83, 1.4),
+ Configuration('conf3', 2.05, 10, 84, 0.8),
+]
+
+def compute_pareto_points(configurations):
+ speedupconfigurations = []
+ energyconfigurations = []
+ #sort configurations based on speedup
+ sorted_configurations = sorted(configurations, key=lambda conf: conf.accuracy_loss)
+
+ start_idx = 0
+ while start_idx < len(sorted_configurations):
+ end_idx = start_idx + 1;
+ # find end_idx
+ while end_idx < len(sorted_configurations) and (sorted_configurations[end_idx].accuracy_loss - sorted_configurations[start_idx].accuracy_loss < AL_THRESHOLD) :
+ end_idx += 1
+ # find best speedup end energy in this accuracy loss level
+ sp = -1.0
+ sp_idx = 0
+ en = -1.0
+ en_idx = 0
+ for i in range(start_idx, end_idx):
+ if sorted_configurations[i].speedup > sp:
+ sp = sorted_configurations[i].speedup
+ sp_idx = i
+ if sorted_configurations[i].energy > en:
+ en = sorted_configurations[i].energy
+ en_idx = i
+ sp_not_dominated = True
+ # if not empty list of configurations
+ if speedupconfigurations:
+ if speedupconfigurations[-1].speedup >= sp:
+ sp_not_dominated = False
+ en_not_dominated = True
+ # if not empty list of configurations
+ if energyconfigurations:
+ if energyconfigurations[-1].energy >= en:
+ en_not_dominated = False
+ if sp_not_dominated:
+ speedupconfigurations.append(sorted_configurations[sp_idx])
+ if en_not_dominated:
+ energyconfigurations.append(sorted_configurations[en_idx])
+ # outer while loop variable increment
+ start_idx = end_idx
+ return [speedupconfigurations, energyconfigurations]
+
+
+def compute_pareto_points_with_margin(configurations, speedup_band_width, energy_band_width):
+ speedupconfigurations = []
+ energyconfigurations = []
+ #sort configurations based on speedup
+ sorted_configurations = sorted(configurations, key=lambda conf: conf.accuracy_loss)
+
+ idx_to_sp_conf_dict = {}
+ idx_to_en_conf_dict = {}
+
+ start_idx = 0
+ while start_idx < len(sorted_configurations):
+ end_idx = start_idx + 1;
+ # find end_idx
+ while end_idx < len(sorted_configurations) and (sorted_configurations[end_idx].accuracy_loss - sorted_configurations[start_idx].accuracy_loss < AL_THRESHOLD) :
+ end_idx += 1
+ # find best speedup end energy in this accuracy loss level
+ sp = -1.0
+ sp_idx = 0
+ en = -1.0
+ en_idx = 0
+ for i in range(start_idx, end_idx):
+ if sorted_configurations[i].speedup > sp:
+ sp = sorted_configurations[i].speedup
+ sp_idx = i
+ if sorted_configurations[i].energy < en:
+ en = sorted_configurations[i].energy
+ en_idx = i
+ sp_not_dominated = True
+ # if not empty list of configurations
+ if speedupconfigurations:
+ if speedupconfigurations[-1].speedup >= sp:
+ sp_not_dominated = False
+ en_not_dominated = True
+ # if not empty list of configurations
+ if energyconfigurations:
+ if energyconfigurations[-1].energy >= en:
+ en_not_dominated = False
+ if sp_not_dominated:
+ speedupconfigurations.append(sorted_configurations[sp_idx])
+ idx_to_sp_conf_dict[start_idx] = len(speedupconfigurations)-1
+ if en_not_dominated:
+ energyconfigurations.append(sorted_configurations[en_idx])
+ idx_to_en_conf_dict[start_idx] = len(energyconfigurations)-1
+ # outer while loop variable increment
+ start_idx = end_idx
+
+ # We want to add configurations in a band of a certain width around the curves
+ # not possible to do during contruction, because the quality of the curve would
+ # deteriorate quickly
+
+ AdjustedSpeedupCurve = []
+ AdjustedEnergyCurve = []
+
+ start_idx = 0
+ while start_idx < len(sorted_configurations):
+ end_idx = start_idx + 1;
+ # find end_idx
+ while end_idx < len(sorted_configurations) and (sorted_configurations[end_idx].accuracy_loss - sorted_configurations[start_idx].accuracy_loss < AL_THRESHOLD) :
+ end_idx += 1
+ for i in range(start_idx, end_idx):
+ if sorted_configurations[i].speedup + speedup_band_width >= speedupconfigurations[idx_to_sp_conf_dict[start_idx]].speedup:
+ AdjustedSpeedupCurve.append(sorted_configurations[i])
+ if sorted_configurations[i].energy + energy_band_width >= energyconfigurations[idx_to_en_conf_dict[start_idx]].energy:
+ AdjustedEnergyCurve.append(sorted_configurations[i])
+ # outer while loop variable increment
+ start_idx = end_idx
+
+ return [AdjustedSpeedupCurve, AdjustedEnergyCurve]
+
+
+
+def findParetoConfigs(base_dir, layer_costs, accuracy):
+
+ result_dir = base_dir + "/pareto/"
+ try:
+ os.mkdir(result_dir)
+ except:
+ print "could not create dir"
+
+ input_dir = base_dir + "/full_results/"
+ #result_dir = "../build_tuner/tuner_results/alexnet_cifar10/loss_3/batch15"
+ config_arr = loadConfigData(input_dir, layer_costs, accuracy)
+
+ config_list = []
+
+ it = 0
+ for config in config_arr:
+ config = Configuration(config.fname , config.speedup, 100, config.avg_accuracy, config.avg_loss)
+ config_list.append(config)
+
+
+ if len(config_list) < 30:
+ SPEEDUP_BAND_SIZE = 1.2
+
+
+ ASC, AEC = compute_pareto_points_with_margin(config_list, SPEEDUP_BAND_SIZE, ENERGY_BAND_SIZE)
+
+
+ print ("len(config_list) = ", len(config_list))
+ print ("len(ASC) = ", len(ASC))
+
+ #print (ASC)
+ #print (config_list)
+
+ for conf in ASC:
+ #dst_path = conf.name.replace("full_results", "pareto")
+ src_path = base_dir + "/full_results/" + conf.name
+ dst_path = base_dir + "/pareto/" + conf.name
+ shutil.copy(src_path, dst_path)
+
+
+
+if __name__ == "__main__":
+
+ get_pareto_configs("")
+
+ #SC, EC = compute_pareto_points(configuration_objects)
+ #ASC, AEC = compute_pareto_points_with_margin(configuration_objects, SPEEDUP_BAND_SIZE, ENERGY_BAND_SIZE)
+
+ #print(SC)
+ #print(EC)
+
+ #print(ASC)
+ #print(AEC)
diff --git a/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/promise_tuner3.py b/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/promise_tuner3.py
index 07a5bf0bcf..87ed35bbc4 100644
--- a/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/promise_tuner3.py
+++ b/llvm/projects/hpvm-tensor-rt/opentuner/autotuner/promise_tuner3.py
@@ -21,9 +21,10 @@ import subprocess
import threading
import psutil
-from measure_confidence2 import dump_promise_confidence_files2
+from measure_confidence2 import dump_promise_confidence_files3
from select_top_results import select_top_results
from time import sleep
+from pareto_curve import findParetoConfigs
layer_file = ""
@@ -48,6 +49,7 @@ def readCostFile(file_path):
cost = float(x.strip())
layer_costs.append(cost)
+ print ("len(layer_costs) = ", layer_costs)
f.close()
@@ -192,7 +194,11 @@ class ClangFlagsTuner(MeasurementInterface):
evaluated_configs[accuracy] = 1
shutil.copy('promise_flags', output_dir + '/' + binary_name + '_' + str(test_id))
-
+ f_acc = open(output_dir + '/' + binary_name + '_' + str(test_id) + "_accuracy", "w")
+ f_acc.write(str(accuracy))
+ f_acc.close()
+
+
print "done with one run"
test_id += 1
@@ -203,12 +209,17 @@ class ClangFlagsTuner(MeasurementInterface):
def save_final_config(self, configuration):
print "Dumping High Confidence results \n"
- sleep(20)
+ sleep(2)
+
+
+ findParetoConfigs(orig_result_dir, layer_costs, accuracy_threshold)
+
+ input_dir = orig_result_dir + "/pareto/"
+ output_dir = orig_result_dir + "/high_confidence/"
# Only dumping files with 95% confidence
- dump_promise_confidence_files2(binary_name, orig_result_dir, layer_file, num_flags, accuracy_threshold, layer_costs, 95)
+ dump_promise_confidence_files3(binary_name, input_dir, output_dir, layer_file, num_flags, accuracy_threshold, layer_costs, 95)
#select_top_results(orig_result_dir + "/high_confidence")
-
"""
--
GitLab