Added python script for printing and plotting parboil benchmark results

llvm/test/VISC/parboil/parboilParser.py

+#!/usr/bin/python
+
+import csv
+import sys
+import os
+import collections
+from optparse import OptionParser
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+default_value = '0.0'
+num_figs = 0
+ 
+# parse a csv file and create a dictionary holding all the data
+def parseCSVFile(filename):
+  # open file
+  file = open(filename, "rb")
+
+  # create the dictionary
+  n = lambda k: None
+  csvDict = collections.defaultdict(
+              lambda: collections.defaultdict(
+                lambda: collections.defaultdict(
+                  lambda: collections.defaultdict(
+                    lambda: collections.defaultdict(lambda: default_value)))))
+
+  # parse file and create dict
+  reader = csv.DictReader(file)
+  try:
+    applicationKey = ''
+    versionKey = ''
+    testKey = ''
+    timerKey = ''
+    categoryKey = ''
+    for row in reader:
+      if row['Application']:
+        applicationKey = row['Application']
+      if row['Version']:
+        versionKey = row['Version']
+      if row['Test']:
+        testKey = row['Test']
+      if row['Timer']:
+        timerKey = row['Timer']
+      if row['Category']:
+        categoryKey = row['Category']
+      csvDict[applicationKey][versionKey][testKey][timerKey][categoryKey] = row['Time in seconds']
+  except csv.Error, e:
+    sys.exit('file %s, line %d: %s' % (filename, reader.line_num, e))
+
+  # close file
+  file.close()
+
+  # check correctness
+  file = open(filename, "rb")
+  reader = csv.DictReader(file)
+  try:
+    applicationKey = ''
+    versionKey = ''
+    testKey = ''
+    timerKey = ''
+    categoryKey = ''
+    for row in reader:
+      if row['Application']:
+        applicationKey = row['Application']
+      if row['Version']:
+        versionKey = row['Version']
+      if row['Test']:
+        testKey = row['Test']
+      if row['Timer']:
+        timerKey = row['Timer']
+      if row['Category']:
+        categoryKey = row['Category']
+      assert(csvDict[applicationKey][versionKey][testKey][timerKey][categoryKey] == row['Time in seconds'])
+  except csv.Error, e:
+    sys.exit('file %s, line %d: %s' % (filename, reader.line_num, e))
+  file.close()
+
+  #print csvDict['a']['b']['c']['d']['e']
+  #print csvDict['sgemm']['visc']['c']['d']['e']
+  #print csvDict['sgemm']['opencl_base']['c']['d']['e']
+  #print csvDict['sgemm']['opencl_base']['small']['d']['e']
+  #print csvDict['sgemm']['opencl_base']['small']['Final']['e']
+  #print csvDict['sgemm']['opencl_base']['small']['Final']['IO']
+
+  #for app in csvDict.iterkeys():
+  #  for v in csvDict[app].iterkeys():
+  #    for test in csvDict[app][v].iterkeys():
+  #      for t in csvDict[app][v][test].iterkeys():
+  #        for cat in csvDict[app][v][test][t].iterkeys():
+  #          time = csvDict[app][v][test][t][cat]
+  #          print (app, v, test, t, cat, time)
+
+  # return dict
+  return csvDict
+
+# returns the name of the opecl version of the given application
+def getOpenCLVersionName(app):
+  return 'opencl_base' if app=='stencil' or app=='sgemm' else 'opencl_nvidia'
+
+# returns a list of available tests for the given application
+# the tests are found based on the opecl version, because the visc
+# might not exist in the dict
+def getTests(app, csvDict):
+  v = getOpenCLVersionName(app)
+  return csvDict[app][v].keys()
+
+# print total execution time for all applications and corresponding
+# test sizes
+def printTotalExecutionTimeTable(csvDict):
+  print 'application,visc,opencl'
+  t1 = 'GenVISC_Timer'
+  t2 = 'Final'
+  cat = 'Timer Wall Time'
+  for app in csvDict.iterkeys():
+    v1 = 'visc'
+    v2 = getOpenCLVersionName(app)
+    tests = getTests(app, csvDict)
+    for test in tests:
+      print "{0:s}-{1:s},{2:s},{3:s}".format(app, test, csvDict[app][v1][test][t1][cat], csvDict[app][v2][test][t2][cat])
+
+def plotTotalExecutionTimeTable(csvDict):
+  global num_figs
+  t1 = 'GenVISC_Timer'
+  t2 = 'Final'
+  cat = 'Timer Wall Time'
+  visc_time = []
+  opencl_time = []
+  axis_values = []
+  n_groups = 0;
+  for app in csvDict.iterkeys():
+    v1 = 'visc'
+    v2 = getOpenCLVersionName(app)
+    tests = getTests(app, csvDict)
+    for test in tests:
+      n_groups += 1
+      visc_time.append(float(csvDict[app][v1][test][t1][cat]))
+      opencl_time.append(float(csvDict[app][v2][test][t2][cat]))
+      axis_values.append(app + '\n' + test);
+
+  bar_width = 0.35
+  opacity = 0.4
+  num_figs += 1
+  fig, ax = plt.subplots(num=num_figs)
+  index = np.arange(n_groups)
+  bars_visc = plt.bar(index, visc_time, bar_width, alpha=opacity, color='b', label='VISC')
+  bars_opencl = plt.bar(index + bar_width, opencl_time, bar_width, alpha=opacity, color='r', label='OpenCL')
+  plt.xlabel('Experiments')
+  plt.ylabel('Total Execution Time (s)')
+  plt.title('Total Execution Time - VISC and OpenCL')
+  plt.xticks(index + bar_width, axis_values)
+  plt.legend(loc='best')
+  plt.tight_layout()
+
+def printTimerDecomposition(csvDict, isVisc):
+  # get apps
+  apps = csvDict.keys()
+
+  # get tests for each app
+  tests = dict()
+  for app in apps:
+    tests[app] = getTests(app, csvDict)
+
+  # list of timer-category pairs
+  if isVisc:
+    timers =[('Final', 'Kernel'), 
+             ('Final', 'Load Program Binary'), 
+             ('Final', 'Argument Unpack'), 
+             ('Final', 'Marshal Arguments'), 
+             ('Final', 'Free Memory'), 
+             ('Final', 'Memory Track'), 
+             ('Final', 'Clear Context'), 
+             ('Final', 'Total GPU Computation'), 
+             ('Final', 'Copy Pointer Arguments'), 
+             ('Final', 'Initialize Context'), 
+             ('Final', 'Read Output'), 
+             ('Final', 'Pthread Create'), 
+             ('Final', 'Copy Scalar Arguments'), 
+             ('Final', 'WorkGroup Size Calculation'), 
+             ('Final', 'IO'), 
+             ('Final', 'Output Pack'), 
+             ('Parboil', 'Mem_Untrack'), 
+             ('Parboil', 'Clear_Ctx'),
+             ('GenVISC_Timer', 'Timer Wall Time')]
+  else: 
+    timers =[('Final', 'Init_Ctx'),
+             ('Final', 'Arg_Unpack'), 
+             ('Final', 'Copy_Scalar'), 
+             ('Final', 'Mem_Track'), 
+             ('Final', 'Driver'), 
+             ('Final', 'Output_Unpack'), 
+             ('Final', 'Arg_Pack'), 
+             ('Final', 'Copy'), 
+             ('Final', 'Compute'), 
+             ('Final', 'Setup'), 
+             ('Final', 'Read_Output'), 
+             ('Final', 'IO'), 
+             ('Final', 'Pthread_Create'), 
+             ('Final', 'Kernel'), 
+             ('Final', 'Mem_Free'), 
+             ('Final', 'Copy Async'), 
+             ('Final', 'Copy_Ptr'),
+             ('Final', 'Output_Pack'),
+             ('Final', 'Mem_Untrack'),
+             ('Final', 'Clear_Ctx'),
+             ('Final', 'Timer Wall Time')]
+
+  line = "Category,"
+  for app in apps:
+    for test in tests[app]:
+      line = line + app + "-" + test + ","
+  print line
+ 
+  for (t, cat) in timers:
+    line = cat + ","
+    for app in apps:
+      v = 'visc' if isVisc else getOpenCLVersionName(app)
+      for test in tests[app]:
+        line = line + csvDict[app][v][test][t][cat] + ","
+    print line 
+
+def plotTimerDecomposition(csvDict, isVisc):
+  global num_figs
+
+  # get apps
+  apps = csvDict.keys()
+
+  # get tests for each app
+  tests = dict()
+  for app in apps:
+    tests[app] = getTests(app, csvDict)
+
+  # list of timer-category pairs
+  if isVisc:
+    timers =[('Final', 'Kernel'), 
+             ('Final', 'Load Program Binary'), 
+             ('Final', 'Argument Unpack'), 
+             ('Final', 'Marshal Arguments'), 
+             ('Final', 'Free Memory'), 
+             ('Final', 'Memory Track'), 
+             ('Final', 'Clear Context'), 
+             ('Final', 'Total GPU Computation'), 
+             ('Final', 'Copy Pointer Arguments'), 
+             ('Final', 'Initialize Context'), 
+             ('Final', 'Read Output'), 
+             ('Final', 'Pthread Create'), 
+             ('Final', 'Copy Scalar Arguments'), 
+             ('Final', 'WorkGroup Size Calculation'), 
+             ('Final', 'IO'), 
+             ('Final', 'Output Pack'), 
+             ('Parboil', 'Mem_Untrack'), 
+             ('Parboil', 'Clear_Ctx'),
+             ('GenVISC_Timer', 'Timer Wall Time')]
+  else: 
+    timers =[('Final', 'Init_Ctx'),
+             ('Final', 'Arg_Unpack'), 
+             ('Final', 'Copy_Scalar'), 
+             ('Final', 'Mem_Track'), 
+             ('Final', 'Driver'), 
+             ('Final', 'Output_Unpack'), 
+             ('Final', 'Arg_Pack'), 
+             ('Final', 'Copy'), 
+             ('Final', 'Compute'), 
+             ('Final', 'Setup'), 
+             ('Final', 'Read_Output'), 
+             ('Final', 'IO'), 
+             ('Final', 'Pthread_Create'), 
+             ('Final', 'Kernel'), 
+             ('Final', 'Mem_Free'), 
+             ('Final', 'Copy Async'), 
+             ('Final', 'Copy_Ptr'),
+             ('Final', 'Output_Pack'),
+             ('Final', 'Mem_Untrack'),
+             ('Final', 'Clear_Ctx'),
+             ('Final', 'Timer Wall Time')]
+
+#  num_experiments = 0;
+#  for app in apps:
+#    for test in tests[app]:
+#      num_experiments += 1
+
+#  exp_count = 0
+  for app in apps:
+    v = 'visc' if isVisc else getOpenCLVersionName(app)
+    for test in tests[app]:
+#      exp_count += 1
+      n_vals = 0;
+      axis_values = []
+      time_decomp = []
+      for (t, cat) in timers:
+        n_vals += 1
+        axis_values.append(cat);
+        time_decomp.append(float(csvDict[app][v][test][t][cat])) 
+
+      opacity = 0.4
+      num_figs += 1
+      fig, ax = plt.subplots(num=num_figs)
+      index = np.arange(n_vals)
+      plt.barh(index, time_decomp, alpha=opacity, color='b', label=v)
+      plt.xlabel('Time (s)')
+      plt.ylabel('Timers')
+      plt.title('Time Decomposition - ' + app + ' ' + test + ' ' + v)
+      plt.yticks(index, axis_values)
+      plt.tight_layout()
+
+
+
+
+# command line options parser
+parser = OptionParser()
+parser.add_option("-f","--file",action="store",type="string",dest="filename")
+parser.add_option("--print-totals",action="store_true",default=False,dest="printTotals")
+parser.add_option("--print-timers-visc",action="store_true",default=False,dest="printTimersVisc")
+parser.add_option("--print-timers-opencl",action="store_true",default=False,dest="printTimersOpencl")
+parser.add_option("--print-all",action="store_true",default=False,dest="printAll")
+parser.add_option("--plot-totals",action="store_true",default=False,dest="plotTotals")
+parser.add_option("--plot-timers-visc",action="store_true",default=False,dest="plotTimersVisc")
+parser.add_option("--plot-timers-opencl",action="store_true",default=False,dest="plotTimersOpencl")
+parser.add_option("--plot-all",action="store_true",default=False,dest="plotAll")
+
+# main
+def main():
+  global num_figs
+  # parse command line arguments
+  (options, args) = parser.parse_args()
+
+  # find out input
+  if not options.filename and not args:
+    print "parboilReader.py: Error: No input file was given!"
+    raise OSError
+  elif not options.filename:
+    filename = args[0]
+  else:
+    filename = options.filename
+
+  # create a dict for the csv file
+  csvDict = parseCSVFile(filename)
+
+  # print tables
+  if options.printTotals or options.printAll:
+    printTotalExecutionTimeTable(csvDict)
+    print ''
+
+  if options.printTimersVisc or options.printAll:
+    printTimerDecomposition(csvDict, isVisc=True)
+    print ''
+
+  if options.printTimersOpencl or options.printAll:
+    printTimerDecomposition(csvDict, isVisc=False)
+    print ''
+
+  # plot graphs
+  if options.plotTotals or options.plotAll:
+    plotTotalExecutionTimeTable(csvDict)
+
+  if options.plotTimersVisc or options.plotAll:
+    plotTimerDecomposition(csvDict, isVisc=True)
+
+  if options.plotTimersOpencl or options.plotAll:
+    plotTimerDecomposition(csvDict, isVisc=False)
+
+  plt.show()
+  for i in range(num_figs):
+    plt.close(i)
+
+
+if __name__ == '__main__':
+  main()