Adding code to dump references files + dumping all configs for comparison with Pareto

llvm/projects/hpvm-tensor-rt/autotuner/tuner_driver_src/buildRtConfig.py

@@ -363,6 +363,7 @@ def buildConfigStr(config, layer_desc):
   return config_str
 
 
+
 # Adjusts for expected loss on unseen dataset
 def adjustDevTimeLoss(loss):
 
@@ -370,12 +371,20 @@ def adjustDevTimeLoss(loss):
   if loss < 0.3:
     loss += 0.4
   else:
-    loss = loss * 1.5
+    loss = loss * 1.4
 
   return loss
     
 
 
+def adjustConfigLosses(configurations):
+
+  for config in configurations:
+    config.avg_loss = adjustDevTimeLoss(config.avg_loss)
+
+  
+
+
 def dumpConfig(layer_desc, config_arrs, result_dir):
 
   f = open(result_dir + "/tuner_pareto_confs_" + batch_id + ".txt", "w+")
@@ -383,7 +392,7 @@ def dumpConfig(layer_desc, config_arrs, result_dir):
   for config in config_arrs:
     f.write("+++++\n")
     f.write("conf" + str(it) + " " + str(config.speedup) + " 0 " + \
-            str(config.avg_accuracy) + " " + str(adjustLoss(config.avg_loss)) + "\n")
+            str(config.avg_accuracy) + " " + str(config.avg_loss) + "\n")
 
     config_str = buildConfigStr(config, layer_desc)
 
@@ -402,7 +411,7 @@ def dumpDevConfigsToRTFile(configurations, config_out_path, bench_layer_composit
   for config in configurations:
     f.write("+++++\n")
     f.write("conf" + str(it) + " " + str(config.speedup) + " 0 " +  \
-            str(config.avg_accuracy) + " " + str(adjustDevTimeLoss(config.avg_loss)) + "\n")
+            str(config.avg_accuracy) + " " + str(config.avg_loss) + "\n")
 
     config_str = buildConfigStr(config, bench_layer_composition)
 

llvm/projects/hpvm-tensor-rt/autotuner/tuner_driver_src/pareto_utils.py

@@ -181,7 +181,8 @@ def dumpParetoConfigsToDir(input_dir, output_dir, gold_accuracy):
     max_speedup = 1.0  # No Speedup since no configuration found
 
   
-  SPEEDUP_BAND_SIZE = 0.05    # max_speedup * 1.0 / 12   # 4  # 20% of the max speedup
+  #SPEEDUP_BAND_SIZE = 0.05    # max_speedup * 1.0 / 12   # 4  # 20% of the max speedup
+  SPEEDUP_BAND_SIZE = max_speedup * 1.0 / 10   # 10% of the max speedup is the 'BAND SIZE'  
   ENERGY_BAND_SIZE = 10
 
   print ("*max_speedup = ", max_speedup)         
@@ -189,9 +190,9 @@ def dumpParetoConfigsToDir(input_dir, output_dir, gold_accuracy):
   
   ASC, AEC = compute_pareto_points_with_margin(config_list, SPEEDUP_BAND_SIZE, ENERGY_BAND_SIZE)
 
-
   # Prevents very small pareto-curves
-  if len(ASC) < 10  or len(config_list) < 20:
+  #if len(ASC) < 10  or len(config_list) < 20:
+  if len(config_list) < 10:
     ASC = config_list    
 
   
@@ -207,9 +208,5 @@ def dumpParetoConfigsToDir(input_dir, output_dir, gold_accuracy):
 
 
 
-
-
-#if len(ASC) > 50:
-#  ASC, AEC = compute_pareto_points_with_margin(config_list, SPEEDUP_BAND_SIZE/4, ENERGY_BAND_SIZE)
  
 

llvm/projects/hpvm-tensor-rt/autotuner/tuner_driver_src/run_devtime_tuner.py

@@ -3,6 +3,7 @@
 import os
 import sys
 import subprocess
+import shutil
 from benchmarks import batch_id
 import utils
 import global_paths  
@@ -11,11 +12,12 @@ import buildRtConfig
 import genPlots
 
 
+
 class DevTimeTuner:
 
   def __init__(self, Bench):
 
-    self.autotuner_runs = 10
+    self.autotuner_runs = 20
     self.binary_path = Bench.promise_binary
     self.num_layers = Bench.num_layers
     self.gold_accuracy = Bench.promise_accuracy
@@ -66,8 +68,7 @@ class DevTimeTuner:
 
   def checkExistingDir(self):
 
-    files_dir = self.result_dir + "/high_confidence/"
-    
+    files_dir = self.result_dir + "/high_confidence/"    
     if os.path.exists(files_dir) and len(os.listdir(files_dir)) >= 1:
       print ("result_dir = ", files_dir, " has existing files \n ")
       user_str = input("Enter 'yes' to override - Enter 'no' to skip and exit \n ")
@@ -77,6 +78,27 @@ class DevTimeTuner:
 
 
         
+  def dumpAllConfigurations(self):
+
+    input_dir = self.result_dir + "/high_confidence/"
+    
+    configurations = buildRtConfig.loadConfigsFromDir(input_dir, self.gold_accuracy)
+    config_out_path = self.result_dir + "dev_all_config.txt"
+    bench_layer_composition = utils.getLayerComposition(self.layer_file)
+
+    sorted_configurations = sorted(configurations, key=lambda conf: conf.avg_loss)
+
+    # Adding an extra loss to tuned configurations - adjusting for unseen data
+    buildRtConfig.adjustConfigLosses(sorted_configurations)  
+    
+    buildRtConfig.dumpDevConfigsToRTFile(sorted_configurations, config_out_path, bench_layer_composition)
+
+    plot_file_path = self.result_dir + "dev_all_conf_plot.png"
+    genPlots.genScatterPlotFromConfigs(sorted_configurations, plot_file_path)
+    
+
+
+                
   def dumpParetoFiles(self):
 
     input_dir = self.result_dir + "/high_confidence/"
@@ -92,15 +114,38 @@ class DevTimeTuner:
 
 
     sorted_configurations = sorted(configurations, key=lambda conf: conf.avg_loss)
+
+    # Adding an extra loss to tuned configurations - adjusting for unseen data
+    buildRtConfig.adjustConfigLosses(sorted_configurations)  
     
-    # FIXIT: Fix Config str - uses FP16 - FP32 combinaion within layer
-    # FIXIT: SORT before printing pareto configs
     buildRtConfig.dumpDevConfigsToRTFile(sorted_configurations, config_out_path, bench_layer_composition)
 
     plot_file_path = self.result_dir + "dev_pareto_plot.png"
     genPlots.genScatterPlotFromConfigs(sorted_configurations, plot_file_path)
+
+
+
+  def dumpReferenceFiles(self):
+
+    ref_dir = self.result_dir + "/references/"
+    utils.createDir(ref_dir)
+
+    sources = {"run_devtime_tuner.py", "benchmarks.py", "buildRtConfig.py", "global_paths.py"}
+
+    for src in sources:
+      src_path = global_paths.tensorRT_dir + "/autotuner/tuner_driver_src/" + src
+      dst_path = ref_dir + "/" + src    
+      shutil.copy(src_path, dst_path)
     
-    
+
+    data_files = {self.cost_file, self.layer_file, self.layer_knobs}
+
+    for datafile in data_files:
+      shutil.copy(datafile, ref_dir)
+  
+       
+
+
 
   def setBaselineAccuracy(self):
 
@@ -115,13 +160,15 @@ class DevTimeTuner:
     self.setBaselineAccuracy()
     
     
-    self.invokeDevTunerScript(0.85, self.autotuner_runs)
-    self.invokeDevTunerScript(1.7, self.autotuner_runs)
-    self.invokeDevTunerScript(2.5, self.autotuner_runs)
+    self.invokeDevTunerScript(0.8, self.autotuner_runs)
+    self.invokeDevTunerScript(1.5, self.autotuner_runs)
+    self.invokeDevTunerScript(2.1, self.autotuner_runs)
   
 
     self.dumpParetoFiles()
-    
+    self.dumpAllConfigurations()
 
+    # NOTE: dumping files for checking experimental parameters for each batch
+    self.dumpReferenceFiles()