Incorporating Pareto Curve for Conf Measurement

llvm/projects/hpvm-tensor-rt/opentuner/autotuner/measure_confidence2.py

@@ -19,8 +19,8 @@ def getAccuracy(file_name):
   return accuracy  
 
 
-total_runs = 60.0
-fails_allowed = 4
+total_runs = 40.0
+fails_allowed = 3
 skip_lines = 0
 
 
@@ -297,6 +297,8 @@ def getConfigCost(layer_costs, config_str):
       continue
 
     orig_cost += layer_costs[it]
+
+    #print ("orig_cost = ", orig_cost, " flag_value = ", flag_value) 
     
     if flag_value == 11:
       total_cost += layer_costs[it]
@@ -307,10 +309,12 @@ def getConfigCost(layer_costs, config_str):
     elif flag_value < 8:
       divisor = 5 + (7 - flag_value)
       total_cost += (layer_costs[it] / divisor)
-      
+ 
     it += 1
-    
-  return total_cost, (orig_cost / total_cost)
+
+  speedup = orig_cost * 1.0 / total_cost * 1.0
+  
+  return total_cost, speedup 
   
 
 
@@ -406,6 +410,7 @@ def dump_promise_confidence_files(binary, result_dir, layer_file_path,
 def dump_promise_confidence_files2(binary, result_dir, layer_file_path,
                                    num_flags, accuracy, layer_costs, confidence):
 
+
   #result_dir = args.result_dir
   output_dir = result_dir + "/high_confidence"
   input_dir = result_dir + "/full_results"
@@ -445,6 +450,49 @@ def dump_promise_confidence_files2(binary, result_dir, layer_file_path,
 
 
 
+
+def dump_promise_confidence_files3(binary, input_dir, output_dir, layer_file_path,
+                                   num_flags, accuracy, layer_costs, confidence):
+
+
+  #result_dir = args.result_dir
+  #output_dir = result_dir + "/high_confidence"
+  #input_dir = result_dir + "/full_results"
+
+  if not os.path.exists(output_dir):
+    os.mkdir(output_dir)    
+
+  layer_sizes = processLayerFile(layer_file_path);
+  print layer_sizes
+  sleep(2)
+    
+  confidence_list = compute_promise_confidence2(binary, accuracy, confidence, layer_costs, input_dir, output_dir)
+  print confidence_list
+
+  # Ascending sort on accuracy
+  sorted_list = sorted(confidence_list, key = lambda tup: tup[1])
+   
+  promise_file = open(output_dir + "/promise_confs.txt", "w+")
+  confidence_file = open(output_dir + "/confidence_summary.txt", "w+")
+
+  max_configs = 50
+  it_count = 0
+  for x in sorted_list:
+    if x[1] > accuracy and x[0] > confidence:
+      config_str = getLayerConfigStr(x[3], layer_sizes, num_flags)
+      promise_file.write(config_str + "\n")
+      it_count += 1
+      if it_count > max_configs:
+        break
+       
+    confidence_file.write(str(x[0]) + "\t" + str(x[1]) + "\t" + str(x[3]) + "\n")    
+    
+  promise_file.close()
+  confidence_file.close()
+  
+  print "Dumped Confidence Summary"
+
+
   
 
 

llvm/projects/hpvm-tensor-rt/opentuner/autotuner/pareto_curve.py

+
+
+import os
+import shutil
+from measure_confidence2 import getConfigCost
+
+
+class Config:
+  def __init__(self):
+    self.avg_accuracy = 0
+    self.avg_loss = 0
+    self.speedup = 1
+    self.fname = ""
+    self.flags = []
+
+
+
+
+def skipFile(fname):
+
+  skip_files = {}
+  skip_files["confidence_summary.txt"] = 1
+  skip_files["promise_confs.txt"] = 1
+
+  if "accuracy" in fname:
+    return True
+
+  if fname in skip_files:
+    return True
+  else:
+    return False
+    
+
+  
+    
+def loadConfigData(result_dir, layer_costs, baseline_accuracy):
+
+  config_arr = []
+  
+  #result_dir += "/promise_tuner/high_confidence/"
+  file_names = os.listdir(result_dir)
+
+  
+  for fname in file_names:
+    if not skipFile(fname):
+
+      fpath = result_dir + fname  
+      config = Config()
+      f = open(fpath, "r")
+
+      config_str = f.read()
+      cost, speedup = getConfigCost(layer_costs, config_str)
+
+      config.speedup = speedup
+      config.fname = fname
+
+      fpath2 = fpath + "_accuracy"
+      f2 = open(fpath2, "r")
+      acc_str = f2.read().strip()
+      accuracy = float(acc_str)
+      
+      config.avg_accuracy = accuracy
+      config.avg_loss = baseline_accuracy - accuracy
+   
+      config_arr.append(config)
+        
+
+  return config_arr      
+
+    
+
+AL_THRESHOLD = 0.1
+SPEEDUP_BAND_SIZE = 0.3
+ENERGY_BAND_SIZE = 10
+
+
+class Configuration:
+    def __init__(self, name, speedup, energy, accuracy, accuracy_loss):
+        self.name = name
+        self.speedup = speedup
+        self.energy = energy
+        self.accuracy = accuracy
+        self.accuracy_loss = accuracy_loss
+    def __repr__(self):
+        return repr((self.name, self.speedup, self.energy, self.accuracy, self.accuracy_loss))
+
+configuration_objects = [
+    Configuration('conf1', 1.05, 15, 85, 1.2),
+    Configuration('conf2', 2.51, 12, 83, 1.4),
+    Configuration('conf3', 2.05, 10, 84, 0.8),
+]
+
+def compute_pareto_points(configurations):
+    speedupconfigurations = []
+    energyconfigurations = []
+    #sort configurations based on speedup
+    sorted_configurations = sorted(configurations, key=lambda conf: conf.accuracy_loss)
+
+    start_idx = 0
+    while start_idx < len(sorted_configurations):
+        end_idx = start_idx + 1;
+        # find end_idx
+        while end_idx < len(sorted_configurations) and (sorted_configurations[end_idx].accuracy_loss - sorted_configurations[start_idx].accuracy_loss < AL_THRESHOLD) :
+            end_idx += 1
+        # find best speedup end energy in this accuracy loss level
+        sp = -1.0
+        sp_idx = 0
+        en = -1.0
+        en_idx = 0
+        for i in range(start_idx, end_idx):
+            if sorted_configurations[i].speedup > sp:
+                sp = sorted_configurations[i].speedup
+                sp_idx = i
+            if sorted_configurations[i].energy > en:
+                en = sorted_configurations[i].energy
+                en_idx = i
+        sp_not_dominated = True
+        # if not empty list of configurations
+        if speedupconfigurations:
+            if speedupconfigurations[-1].speedup >= sp:
+                sp_not_dominated = False
+        en_not_dominated = True
+        # if not empty list of configurations
+        if energyconfigurations:
+            if energyconfigurations[-1].energy >= en:
+                en_not_dominated = False
+        if sp_not_dominated:
+            speedupconfigurations.append(sorted_configurations[sp_idx])
+        if en_not_dominated:
+            energyconfigurations.append(sorted_configurations[en_idx])
+        # outer while loop variable increment
+        start_idx = end_idx
+    return [speedupconfigurations, energyconfigurations]
+
+
+def compute_pareto_points_with_margin(configurations, speedup_band_width, energy_band_width):
+    speedupconfigurations = []
+    energyconfigurations = []
+    #sort configurations based on speedup
+    sorted_configurations = sorted(configurations, key=lambda conf: conf.accuracy_loss)
+
+    idx_to_sp_conf_dict = {}
+    idx_to_en_conf_dict = {}
+
+    start_idx = 0
+    while start_idx < len(sorted_configurations):
+        end_idx = start_idx + 1;
+        # find end_idx
+        while end_idx < len(sorted_configurations) and (sorted_configurations[end_idx].accuracy_loss - sorted_configurations[start_idx].accuracy_loss < AL_THRESHOLD) :
+            end_idx += 1
+        # find best speedup end energy in this accuracy loss level
+        sp = -1.0
+        sp_idx = 0
+        en = -1.0
+        en_idx = 0
+        for i in range(start_idx, end_idx):
+            if sorted_configurations[i].speedup > sp:
+                sp = sorted_configurations[i].speedup
+                sp_idx = i
+            if sorted_configurations[i].energy < en:
+                en = sorted_configurations[i].energy
+                en_idx = i
+        sp_not_dominated = True
+        # if not empty list of configurations
+        if speedupconfigurations:
+            if speedupconfigurations[-1].speedup >= sp:
+                sp_not_dominated = False
+        en_not_dominated = True
+        # if not empty list of configurations
+        if energyconfigurations:
+            if energyconfigurations[-1].energy >= en:
+                en_not_dominated = False
+        if sp_not_dominated:
+            speedupconfigurations.append(sorted_configurations[sp_idx])
+        idx_to_sp_conf_dict[start_idx] = len(speedupconfigurations)-1
+        if en_not_dominated:
+            energyconfigurations.append(sorted_configurations[en_idx])
+        idx_to_en_conf_dict[start_idx] = len(energyconfigurations)-1
+        # outer while loop variable increment
+        start_idx = end_idx
+
+    # We want to add configurations in a band of a certain width around the curves
+    # not possible to do during contruction, because the quality of the curve would
+    # deteriorate quickly
+
+    AdjustedSpeedupCurve = []
+    AdjustedEnergyCurve = []
+
+    start_idx = 0
+    while start_idx < len(sorted_configurations):
+        end_idx = start_idx + 1;
+        # find end_idx
+        while end_idx < len(sorted_configurations) and (sorted_configurations[end_idx].accuracy_loss - sorted_configurations[start_idx].accuracy_loss < AL_THRESHOLD) :
+            end_idx += 1
+        for i in range(start_idx, end_idx):
+            if sorted_configurations[i].speedup + speedup_band_width >= speedupconfigurations[idx_to_sp_conf_dict[start_idx]].speedup:
+                AdjustedSpeedupCurve.append(sorted_configurations[i])
+            if sorted_configurations[i].energy + energy_band_width >= energyconfigurations[idx_to_en_conf_dict[start_idx]].energy:
+                AdjustedEnergyCurve.append(sorted_configurations[i])
+        # outer while loop variable increment
+        start_idx = end_idx
+
+    return [AdjustedSpeedupCurve, AdjustedEnergyCurve]
+
+
+
+def findParetoConfigs(base_dir, layer_costs, accuracy):
+
+  result_dir = base_dir + "/pareto/"
+  try:
+      os.mkdir(result_dir)
+  except:
+      print "could not create dir"
+
+  input_dir = base_dir + "/full_results/"    
+  #result_dir = "../build_tuner/tuner_results/alexnet_cifar10/loss_3/batch15"
+  config_arr = loadConfigData(input_dir, layer_costs, accuracy)
+
+  config_list = []
+
+  it = 0
+  for config in config_arr:
+    config = Configuration(config.fname , config.speedup, 100, config.avg_accuracy, config.avg_loss)
+    config_list.append(config)
+
+
+  if len(config_list) < 30:
+    SPEEDUP_BAND_SIZE = 1.2
+    
+    
+  ASC, AEC = compute_pareto_points_with_margin(config_list, SPEEDUP_BAND_SIZE, ENERGY_BAND_SIZE)
+
+  
+  print ("len(config_list) = ", len(config_list))
+  print ("len(ASC) = ", len(ASC))
+
+  #print (ASC)
+  #print (config_list)
+
+  for conf in ASC:
+    #dst_path = conf.name.replace("full_results", "pareto")
+    src_path = base_dir + "/full_results/" + conf.name
+    dst_path = base_dir + "/pareto/" + conf.name
+    shutil.copy(src_path, dst_path)
+    
+  
+
+if __name__ == "__main__":
+
+  get_pareto_configs("")
+  
+  #SC, EC = compute_pareto_points(configuration_objects)
+  #ASC, AEC = compute_pareto_points_with_margin(configuration_objects, SPEEDUP_BAND_SIZE, ENERGY_BAND_SIZE)
+
+  #print(SC)
+  #print(EC)
+
+  #print(ASC)
+  #print(AEC)

llvm/projects/hpvm-tensor-rt/opentuner/autotuner/promise_tuner3.py

@@ -21,9 +21,10 @@ import subprocess
 import threading
 import psutil
 
-from measure_confidence2 import dump_promise_confidence_files2
+from measure_confidence2 import dump_promise_confidence_files3
 from select_top_results import select_top_results
 from time import sleep
+from pareto_curve import findParetoConfigs
 
 
 layer_file = ""
@@ -48,6 +49,7 @@ def readCostFile(file_path):
     cost = float(x.strip())
     layer_costs.append(cost)
 
+  print ("len(layer_costs) = ", layer_costs)
   f.close()
   
   
@@ -192,7 +194,11 @@ class ClangFlagsTuner(MeasurementInterface):
       evaluated_configs[accuracy] = 1
       shutil.copy('promise_flags', output_dir + '/' + binary_name + '_' + str(test_id))
 
-        
+      f_acc = open(output_dir + '/' + binary_name + '_' + str(test_id) + "_accuracy", "w")
+      f_acc.write(str(accuracy))
+      f_acc.close()
+                   
+      
     print "done with one run"
 
     test_id += 1
@@ -203,12 +209,17 @@ class ClangFlagsTuner(MeasurementInterface):
   def save_final_config(self, configuration):
 
     print "Dumping High Confidence results \n"
-    sleep(20)
+    sleep(2)
+
+
+    findParetoConfigs(orig_result_dir, layer_costs, accuracy_threshold)
+
+    input_dir = orig_result_dir + "/pareto/"
+    output_dir = orig_result_dir + "/high_confidence/"
     
     # Only dumping files with 95% confidence
-    dump_promise_confidence_files2(binary_name, orig_result_dir, layer_file, num_flags, accuracy_threshold, layer_costs, 95)
+    dump_promise_confidence_files3(binary_name, input_dir, output_dir, layer_file, num_flags, accuracy_threshold, layer_costs, 95)
     #select_top_results(orig_result_dir + "/high_confidence")
-
   
     
     """