Adding opentuner script for PROMISE swing tuning

llvm/projects/hpvm-tensor-rt/opentuner/autotuner/measure_confidence2.py

@@ -3,6 +3,7 @@ import argparse
 import os
 import sys
 import subprocess
+from time import sleep
 
 
 def getAccuracy(file_name):
@@ -18,7 +19,7 @@ def getAccuracy(file_name):
   return accuracy  
 
 
-total_runs = 100.0
+total_runs = 2.0
 skip_lines = 0
 
 
@@ -30,6 +31,7 @@ def test_func():
 def do_multiple_runs(binary_name, accuracy_threshold, confidence_threshold):
 
   successful_runs = 0.0
+  unsuccessful_runs = 0.0
   total_acc = 0
 
   for i in range(int(total_runs)):
@@ -42,6 +44,12 @@ def do_multiple_runs(binary_name, accuracy_threshold, confidence_threshold):
 
     if accuracy > accuracy_threshold:
       successful_runs += 1
+    else:
+      unsuccessful_runs += 1
+
+    if unsuccessful_runs > 6:
+      break
+      
 
   confidence = (successful_runs / total_runs) * 100.0    
   print("confidence = ", confidence)    
@@ -118,9 +126,155 @@ def dump_high_confidence_files(binary, result_dir, accuracy, confidence):
 
   output_file.close()
   print  "Dumped Confidence Summary"
+
+
+  
+
+def processLayerFile(layer_file_path):
+
+  layer_sizes = []
+  layer_file = open(layer_file_path, "r")
+
+  for layer_desc in layer_file:
+    try:
+      toks = layer_desc.split(",")
+      if len(toks) < 2: # Not layer size description
+        continue
+      
+      size = int(toks[1])
+      if "NML" in layer_desc:
+        size = -1
+      layer_sizes.append(size)
+    except:
+      return layer_sizes
+
+  return layer_sizes
+
+
+
+def getLayerConfigStr(config_str, layer_sizes, num_flags):
+
+  new_config_str = ""
+  config_vals = config_str.split(',')
+  it_count = 0
+  layer_count = 0
+  
+  #for layer_size in  val in config_vals:
+  for layer_depth_size in layer_sizes:
+
+    if layer_depth_size == -1:
+      new_config_str += "8"
+      layer_count += 1
+      if layer_count < len(layer_sizes):
+        new_config_str += ","
+      continue
+    
+    val = config_vals[it_count]      
+    if val == "":
+      continue
+    
+    config_val = int(val)
+    # For FP32 and FP32 values, each tensor op needs to be annotated
+    if config_val == 8 or config_val == 9:
+      for i in range(layer_depth_size - 1):
+        new_config_str += val + " "
+      new_config_str += val
+      if layer_count < len(layer_sizes) - 1:
+        new_config_str += ","
+    else:
+      new_config_str += val
+      if layer_count < len(layer_sizes) - 1:
+        new_config_str += ","
+
+    it_count += 1
+    layer_count += 1
+    
+
+  return new_config_str
+
+
+
+def compute_promise_confidence(binary_name, accuracy, confidence, result_dir, output_dir):
+
+  confidence_list = []
+  
+  if not os.path.exists(result_dir):
+    print("Path does not exist")
+    sys.exit(0)
+
+  file_names = os.listdir(result_dir)
+  print file_names
+
+  for file_name in file_names:
+    # Skip sub-directories
+    if os.path.isdir(result_dir + "/" + file_name):
+      continue
+    
+    f = open(result_dir + "/" + file_name)
+    tuner_file = open("promise_flags", "w+")
+
+    config_str = f.read()
+    tuner_file.write(config_str)  
+    tuner_file.close()
+    
+    run_confidence, avg_accuracy = do_multiple_runs(binary_name, accuracy, confidence)
+
+    if run_confidence > 90:
+      f2 = open(output_dir + "/" + file_name, "w+")
+      f2.write("total_runs=" + str(total_runs) + "\t confidence=" + str(run_confidence) + "\t avg_accuracy=" + str(avg_accuracy) + "\n")
+      f2.write(config_str)
+      f2.close()
+
+    flags_str = config_str.replace('\n', ',')
+    conf_result = (run_confidence, avg_accuracy, file_name, flags_str)
+    confidence_list.append(conf_result) 
+
+  return confidence_list
+
+
+
+def dump_promise_confidence_files(binary, result_dir, layer_file_path,
+                                  num_flags, accuracy, confidence):
+
+  #result_dir = args.result_dir
+  output_dir = result_dir + "/high_confidence"
+  input_dir = result_dir + "/full_results"
+
+  if not os.path.exists(output_dir):
+    os.mkdir(output_dir)    
+
+  layer_sizes = processLayerFile(layer_file_path);
+  print layer_sizes
+  sleep(2)
+    
+  confidence_list = compute_promise_confidence(binary, accuracy, confidence, input_dir, output_dir)
+  print confidence_list
+
+  # Ascending sort on accuracy
+  sorted_list = sorted(confidence_list, key = lambda tup: tup[1])
+   
+  promise_file = open(output_dir + "/promise_confs.txt", "w+")
+  confidence_file = open(output_dir + "/confidence_summary.txt", "w+")
+
+  max_configs = 50
+  it_count = 0
+  for x in sorted_list:
+    if x[1] > accuracy and x[0] > confidence:
+      config_str = getLayerConfigStr(x[3], layer_sizes, num_flags)
+      promise_file.write(config_str + "\n")
+      it_count += 1
+      if it_count > max_configs:
+        break
+       
+    confidence_file.write(str(x[0]) + "\t" + str(x[1]) + "\t" + str(x[3]) + "\n")    
+    
+  promise_file.close()
+  confidence_file.close()
   
+  print "Dumped Confidence Summary"
 
   
+  
 
 
 

llvm/projects/hpvm-tensor-rt/opentuner/autotuner/promise_tuner2.py

+#!/usr/bin/env python
+#
+# Optimize blocksize of apps/mmm_block.cpp
+#
+# This is an extremely simplified version meant only for tutorials
+#
+import adddeps  # fix sys.path
+
+import argparse
+import opentuner
+from opentuner import ConfigurationManipulator
+from opentuner import MeasurementInterface
+from opentuner import Result
+from opentuner import EnumParameter
+from opentuner.search.objective import ThresholdAccuracyMinimizeTime
+from opentuner.measurement.inputmanager import FixedInputManager
+import shutil
+import os
+import sys
+import subprocess
+import threading
+import psutil
+
+from measure_confidence2 import dump_promise_confidence_files
+from select_top_results import select_top_results
+from time import sleep
+
+
+layer_file = ""
+output_dir = ""
+flag_ranges = []
+tuning_flags = []
+binary_name = ""
+accuracy_threshold = 10.0
+evaluated_configs = {}
+orig_result_dir = ""
+gpu_layers = 0
+
+
+def getAccuracy(file_name):
+  
+  file = open(file_name, "r")
+  acc_str = file.read()
+  file.close()
+  accuracy = float(acc_str)
+  
+  try:
+    accuracy = float(acc_str)
+  except:
+    return 20
+    
+  print accuracy
+  return accuracy
+    
+
+
+def createFlagsFile(file_name, cfg):
+
+  f = open(file_name, "w+")
+  cmd_config = ""
+  for flag in tuning_flags:
+    flag_value = cfg[flag]
+    cmd_config += str(flag_value) + "\n"
+    
+  f.write(cmd_config)
+  f.close()
+
+
+class ClangFlagsTuner(MeasurementInterface):
+
+  def __init__(self, args):
+    objective = ThresholdAccuracyMinimizeTime(accuracy_threshold)
+    input_manager = FixedInputManager(size=num_flags)
+    self.configs_list = []
+
+    super(ClangFlagsTuner, self).__init__(
+        args, program_name=args.binary,
+        program_version=self.file_hash(args.binary),
+        input_manager=input_manager, objective=objective)
+
+
+    
+
+  def manipulator(self):
+    """
+    Define the search space by creating a
+    ConfigurationManipulator
+    """
+    manipulator = ConfigurationManipulator()
+
+    # NOTE: Skipping first 'gpu_layers' to run on GPU
+    for flag in tuning_flags[:gpu_layers]:
+      manipulator.add_parameter(
+        EnumParameter(flag, [8, 9]))
+      
+    for flag in tuning_flags[gpu_layers:]:
+      manipulator.add_parameter(
+        EnumParameter(flag, flag_ranges
+                      )) #default is needed, optimizations don't work without it(tried and tested)
+    return manipulator
+
+  
+  def run(self, desired_result, input, limit):
+
+    """
+    Run  a given configuration then
+    return performance
+    """
+    cfg = desired_result.configuration.data
+    
+    # NOTE: creates the file with flags read by the runtime
+    createFlagsFile("promise_flags", cfg)
+    
+    run_cmd = binary_name
+    print "binary_name = ", run_cmd
+    #run_result_call_program = self.call_program(run_cmd)
+    #print "returned \n\n"
+
+    FNULL = open(os.devnull, 'wb')
+    p = subprocess.Popen(run_cmd, stdout = FNULL)
+    p.wait()
+
+       
+    accuracy = getAccuracy("final_accuracy")
+    total_comps = abs(accuracy_threshold - accuracy)
+    
+    
+    Result = opentuner.resultsdb.models.Result()
+    Result.time = total_comps
+    Result.accuracy = accuracy
+
+    if accuracy > accuracy_threshold:
+      #if accuracy not in evaluated_configs:
+      config_tuple = (total_comps, accuracy, cfg)
+      self.configs_list.append(config_tuple)
+      evaluated_configs[accuracy] = 1
+      shutil.copy('promise_flags', output_dir + '/' + binary_name + '_' + str(accuracy))
+
+        
+    print "done with one run"
+    
+    return Result
+
+
+  def save_final_config(self, configuration):
+
+    print "Dumping High Confidence results \n"
+    sleep(20)
+    
+    # Only dumping files with 95% confidence
+    dump_promise_confidence_files(binary_name, orig_result_dir, layer_file, num_flags, accuracy_threshold, 95)
+    #select_top_results(orig_result_dir + "/high_confidence")
+
+  
+    
+    """
+    called at the end of autotuning with the best resultsdb.models.Configuration
+    """
+    print "Final configuration", configuration.data
+
+    return
+
+  
+    
+
+
+if __name__ == '__main__':
+
+  argparser = argparse.ArgumentParser(parents=opentuner.argparsers())
+  argparser.add_argument('--binary', help='path to target binary')
+  argparser.add_argument('--num-flags', type=int, help='num of flags to tune')
+  argparser.add_argument('--error-range', type=int, help='range of error values used in tuning') 
+  argparser.add_argument('--accuracy', type=float, help='accuracy threshold')
+  argparser.add_argument('--result-dir', help='result directory')
+  argparser.add_argument('--layer-file', help='layer description')
+  argparser.add_argument('--gpu-layers', type=int, help='first N layers to run on GPU')
+
+  
+  args = argparser.parse_args()
+  binary_name = str(args.binary)
+  print("binary_name = ", binary_name)
+  num_flags = int(args.num_flags)
+  error_range = int(args.error_range)
+  accuracy_threshold = float(args.accuracy)
+  print("accuracy = ", accuracy_threshold)
+  result_dir = args.result_dir
+  orig_result_dir = result_dir
+  if result_dir == "":
+    print("Provide --result-dir ")
+    
+  gpu_layers = args.gpu_layers     
+
+    
+  output_dir = result_dir + "/full_results"
+  print output_dir
+  if not os.path.exists(result_dir):
+    os.mkdir(result_dir)
+    
+  if not os.path.exists(output_dir):
+    print("Creating output directory = ", output_dir)
+    os.mkdir(output_dir)
+
+  for j in range(error_range):
+    flag_ranges.append(j)
+
+  print("flag_ranges = ", flag_ranges)
+
+  # File with layer description
+  layer_file = args.layer_file
+  
+  for i in range(num_flags):
+    tuning_flags.append("flag" + str(i))
+  
+  ClangFlagsTuner.main(argparser.parse_args())