diff --git a/llvm/projects/hpvm-tensor-rt/build_pldi/table_generator.py b/llvm/projects/hpvm-tensor-rt/build_pldi/table_generator.py
index 5d4aec4b94e6bd2f1e316b9fa5435d772f0c6c0f..c610aa658c9e5fa8f8890d09a6638b14d1036c0f 100644
--- a/llvm/projects/hpvm-tensor-rt/build_pldi/table_generator.py
+++ b/llvm/projects/hpvm-tensor-rt/build_pldi/table_generator.py
@@ -6,59 +6,102 @@ import shutil
from collections import defaultdict
class TableGenerator:
-
+
+ '''
+ Stores all precision conversions used.
+ '''
precision_conversions = frozenset(["h2f", "f2h"])
- def __init__(self, dir_name, iters, profiler_binary_name):
- self.__dir_name = dir_name
+ def __init__(self, dir_path, iters, profiler_binary_name):
+ '''
+ Args:
+ dir_path: Path of directory containing network binaries
+ iters: Number of iterations to run each binary for
+ profiler_binary_name: Name of offline profiler binary to run
+ '''
+ self.__dir_path = dir_path
- # directory/path/network_name (last item in path)
- self.__network_name = os.path.split(dir_name)[-1]
+ # Name of the actual directory
+ self.__network_name = os.path.split(dir_path)[-1]
self.__iters = iters
self.__profiler_binary_name = profiler_binary_name
- self.__results_dir_name = "%s_results" % self.__dir_name
+ # Path to results directory
+ self.__results_dir_path = "%s_results" % self.__dir_path
+
+ # Outputted table file
self.__table_filename = "%s_tensors.txt" % self.__network_name
- def __is_binary(self, file_path):
- # Binary name must start with the network name as per our naming standards
+ # Nested default dictionary of default dicts
+ self.__table = self.__build_nested_default_dict()
+
+
+ def generate_table(self):
+ '''
+ Generates a table file called <network_name>_tensors.txt in the
+ '''
+ self.__build_internal_table()
+ self.__output_table()
+
+
+ def __should_execute_file(self, file_path):
+ '''
+ Checks if the file at the given file path is a binary that should be run
+ by the profiler. Must exist, be a binary, and must start with the network
+ name as per our naming standards.
+
+ Args:
+ file_path: Path of the file to check
+ '''
return os.path.isfile(file_path) and os.access(file_path, os.X_OK) and \
file_path.find(self.__network_name) != -1
- def run_binaries_in_input_dir(self):
- if not os.path.isdir(self.__dir_name):
- print("ERROR: Directory %s not found" % self.__dir_name)
+ def run_inputted_binaries(self):
+ '''
+ Invokes the profiler to run all appropriate binaries (must start with the network
+ name) in the inputted directory. Result files generated by the profiler are
+ stored in the results file directory and are named <binary_name>.txt. These results
+ files are then parsed in a later step to generate the table
+ '''
+ if not os.path.isdir(self.__dir_path):
+ print("ERROR: Directory %s not found" % self.__dir_path)
exit(1)
try:
- os.mkdir(self.__results_dir_name)
+ os.mkdir(self.__results_dir_path)
except OSError:
- if os.path.isdir(self.__results_dir_name):
+ if os.path.isdir(self.__results_dir_path):
print("Directory already exists. Clearing directory.")
- for old_file in glob.glob(os.path.join(self.__results_dir_name, "*")):
+ for old_file in glob.glob(os.path.join(self.__results_dir_path, "*")):
os.remove(old_file)
else:
print("ERROR: Directory doesn't exist but failed to create dir")
- for binary_name in os.listdir(self.__dir_name):
- binary_path = os.path.join(self.__dir_name, binary_name)
+ for binary_name in os.listdir(self.__dir_path):
+ binary_path = os.path.join(self.__dir_path, binary_name)
- if not self.__is_binary(binary_path):
+ if not self.__should_execute_file(binary_path):
continue
- if not os.path.isfile(binary_path):
- print("ERROR: Binary %s not found" % binary_path)
- exit(1)
-
- output_file = os.path.join(self.__results_dir_name, binary_name + ".txt")
+ output_file = os.path.join(self.__results_dir_path, binary_name + ".txt")
# No stdout/stderr piping needed for now
subprocess.Popen([profiler_binary_name, binary_path, str(self.__iters), \
output_file]).communicate()
def __get_approximation_type(self, results_filename):
+ '''
+ Parses a given results filename for the approximation type.
+ Format assumption: <network_name>_<approx_type>.txt
+
+ Args:
+ results_filename: Name of results file
+
+ Returns:
+ the approximation technique (ex: fp16)
+ '''
approx_type_start_ind = results_filename.find(self.__network_name) \
+ len(self.__network_name) + 1 # + 1 to account for _ delimiter
approx_type_end_ind = results_filename.find(".txt")
@@ -66,34 +109,54 @@ class TableGenerator:
def __parse_tensor_operation_line(self, tensor_op_line):
- print(tensor_op_line)
+ '''
+ Parses a tensor operation line (within a output file from the offline
+ profiler for the operation name, the total time used, and the total
+ energy used
+
+ Args:
+ tensor_op_line: Tensor operation line from output file
+
+ Returns:
+ operation name
+ total time used
+ total energy used
+ '''
line_as_list = tensor_op_line.split(",")
return line_as_list[0], line_as_list[1], line_as_list[2]
def __build_nested_default_dict(self):
+ '''
+ Builds a nested default dictionary with an arbitrary number of levels
+ '''
return defaultdict(self.__build_nested_default_dict)
- # h2f or f2h
def __get_original_operation_name(self, op_name):
+ '''
+ Parses an operation name containing _<conversion type> for the original
+ operation name.
+ Format assumption: <original_op_name>_<conversion type>
+
+ Args:
+ op_name: Name of the operation
+
+ Returns:
+ the original operation name
+ '''
underscore_ind = op_name.find("_")
return op_name[ : underscore_ind], op_name[underscore_ind + 1 : ]
- def generate_table(self):
- self.__table = self.__build_nested_default_dict()
- self.__build_internal_table()
- self.__output_table()
-
def __build_internal_table(self):
- for results_file_name in os.listdir(self.__results_dir_name):
+ for results_file_name in os.listdir(self.__results_dir_path):
# Ignore if it's not a results file
if results_file_name == self.__table_filename or \
not results_file_name.startswith(self.__network_name):
continue
approx_type = self.__get_approximation_type(results_file_name)
- results_file = open(os.path.join(self.__results_dir_name, results_file_name), "r")
+ results_file = open(os.path.join(self.__results_dir_path, results_file_name), "r")
for line in results_file:
line = line.strip()
@@ -122,7 +185,7 @@ class TableGenerator:
results_file.close()
def __output_table(self):
- table_file_path = os.path.join(self.__results_dir_name, self.__table_filename)
+ table_file_path = os.path.join(self.__results_dir_path, self.__table_filename)
# TODO un hard code this
soc_operations_file_name = os.path.join("/home/nvidia/soc_simulator", "%s_cifar10" % self.__network_name, "%s_ops.txt" % self.__network_name)
@@ -184,9 +247,9 @@ class TableGenerator:
# Try doing this per layer first
pass
-binary_dir_name = "/home/nvidia/Gitlab/hpvm/llvm/projects/hpvm-tensor-rt/build_pldi/mobilenet"
+binary_dir_path = "/home/nvidia/Gitlab/hpvm/llvm/projects/hpvm-tensor-rt/build_pldi/mobilenet"
num_iters = 1
profiler_binary_name = "/home/nvidia/awesome_profiler/pp"
-table_gen = TableGenerator(binary_dir_name, num_iters, profiler_binary_name)
-#table_gen.run_binaries_in_input_dir()
-table_gen.generate_table()
+table_gen = TableGenerator(binary_dir_path, num_iters, profiler_binary_name)
+#table_gen.run_inputted_binaries()
+#table_gen.generate_table()