diff --git a/hpvm/projects/hpvm-profiler/hpvm_profiler/__init__.py b/hpvm/projects/hpvm-profiler/hpvm_profiler/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e007ca9277f9e584708488ee57fd08c693a00279
--- /dev/null
+++ b/hpvm/projects/hpvm-profiler/hpvm_profiler/__init__.py
@@ -0,0 +1,175 @@
+from pathlib import Path
+from typing import Iterable, List, Tuple, Union
+from dataclasses import dataclass
+
+PathLike = Union[Path, str]
+
+
+def profile_configs(
+ binary_path: PathLike,
+ config_path: PathLike,
+ output_config_path: PathLike,
+ profile_filename: str = "profile_info.txt",
+ qos_filename: str = "final_accuracy",
+):
+ """
+ Profile an HPVM configuration file with an HPVM binary.
+ The configuration file must have the baseline as the first configuration.
+
+ binary_path: Union[Path, str]
+ Path to binary to be executed in profiling.
+ config_path: Union[Path, str]
+ Path to config file (HPVM configuration format)
+ with configs to enumerate for profiling.
+ output_config_path: Union[Path, str]
+ Path where the output configs are written.
+ The output config file has the same configs as the input `config_path` file,
+ but the performance and energy readings are updated.
+ profile_filename: str
+ Name of profile file generated by the binary (in current directory).
+ This defaults to "profile_info.txt" and should not be changed for HPVM binaries.
+ qos_filename: str
+ Name of QoS file generated by the binary (in current directory).
+ It contains a single float number as the QoS of this run.
+ This defaults to "final_accuracy" and should not be changed for HPVM binaries.
+ """
+
+ from subprocess import check_call
+ from tempfile import NamedTemporaryFile
+
+ # Read first line ("the float") and configs in config file
+ header, configs = read_hpvm_configs(Path(config_path))
+ if not configs:
+ raise ValueError("Config file with no configs is unsupported.")
+ temp_file = NamedTemporaryFile("w")
+ baseline_time, baseline_acc = None, None
+ for idx, config in enumerate(configs):
+ # Write config to temp config file
+ write_hpvm_config(header, [config], Path(temp_file.name))
+ # Run binary_path binary,
+ # which generates `profile_filename` and `qos_filename` file in cwd.
+ check_call(str(binary_path))
+ # Read these two files for time and QoS info.
+ time = _read_profile_file(Path(profile_filename))
+ acc = _read_qos_file(Path(qos_filename))
+ if idx == 0:
+ baseline_time, baseline_acc = time, acc
+ continue
+ assert baseline_time is not None and baseline_acc is not None
+ speedup = baseline_time / time
+ config.update_profile_results(speedup, acc, baseline_acc)
+ write_hpvm_config(header, configs, Path(output_config_path))
+ temp_file.close()
+
+
+def plot_hpvm_configs(
+ config_path: PathLike,
+ save_to: PathLike = None,
+ show_qos_loss: bool = True,
+ **fig_kwargs,
+):
+ """
+ Plot the QoS-speedup information in an HPVM configuration file.
+ It is recommended to profile the config file first (using `profile_configs`)
+ to obtain real speedup numbers.
+ This function creates a `matplotlib.pyplot.Figure`, plots on it, and returns it.
+
+ config_path: Union[Path, str]
+ Path to the config file (HPVM configuration format).
+ save_to: Union[Path, str]
+ File to save figure into. Default is None: don't save figure (just return it).
+ show_qos_loss: bool
+ Show the loss of QoS on x axis of the figure. Defaults to True.
+ If False, will use (absolute) QoS instead of QoS loss.
+ fig_kwargs:
+ Arguments to pass to `plt.subplots`.
+ """
+
+ import numpy as np
+ import matplotlib.pyplot as plt
+
+ _, configs = read_hpvm_configs(config_path)
+ get_qos = lambda c: c.qos_loss if show_qos_loss else c.qos
+ qos_speedup = np.array([(get_qos(c), c.speedup) for c in configs])
+ qoses, speedups = qos_speedup.T
+ fig, ax = plt.subplots(**fig_kwargs)
+ ax.scatter(qoses, speedups)
+ ax.xlabel("QoS Loss")
+ ax.ylabel("Speedup (X)")
+ if save_to:
+ fig.savefig(save_to, dpi=300)
+ return fig
+
+@dataclass
+class Config:
+ conf_name: str
+ speedup: float
+ energy: float
+ qos: float
+ qos_loss: float
+ # We don't care about the information in this part, and we don't parse this.
+ config_body: List[str]
+
+ def update_profile_results(self, speedup: float, qos: float, base_qos: float):
+ recorded_base_qos = self.qos + self.qos_loss
+ if abs(recorded_base_qos - base_qos) > 1e-3:
+ raise ValueError(
+ f"Baseline QoS mismatch. Original: {recorded_base_qos}, measured: {base_qos}"
+ )
+ self.speedup = speedup
+ self.qos = qos
+ self.qos_loss = base_qos - qos
+
+ def __repr__(self) -> str:
+ header_fields = [
+ self.conf_name,
+ self.speedup,
+ self.energy,
+ self.qos,
+ self.qos_loss,
+ ]
+ header = " ".join(str(field) for field in header_fields)
+ return f"{header}\n{self.config_body}"
+
+ __str__ = __repr__
+
+
+def read_hpvm_configs(config_file: PathLike) -> Tuple[str, List[Config]]:
+ # def read_hpvm_configs(config_file, config_num, temp_file):
+ ret_configs = []
+ with open(config_file) as f:
+ text = f.read()
+ opening, closing = "+++++", "-----"
+ # There's 1 float sitting on the first line of config file.
+ # We don't use it, but want to keep that intact.
+ header, *configs = text.split(opening)
+ header = header.strip()
+ for config_text in configs:
+ config_text = config_text.replace(closing, "").strip()
+ config_header, *config_body = config_text.splitlines()
+ conf_name, *number_fields = config_header.split(" ")
+ speedup, energy, qos, qos_drop = [float(s) for s in number_fields]
+ ret_configs.append(
+ Config(conf_name, speedup, energy, qos, qos_drop, config_body)
+ )
+ return header, ret_configs
+
+
+def write_hpvm_config(header: str, configs: Iterable[Config], to_file: PathLike):
+ text_segs = [header] + [str(config) for config in configs]
+ with open(to_file, "w") as f:
+ f.write("\n".join(text_segs))
+
+
+def _read_profile_file(profile_file_path: Path):
+ with profile_file_path.open() as f:
+ target_lines = [line.strip() for line in f if "Total Time" in line]
+ if len(target_lines) != 1:
+ raise RuntimeError(f"Profile {profile_file_path} malformed")
+ (target_line,) = target_lines
+ return float(target_line.split()[3])
+
+
+def _read_qos_file(qos_file_path: Path):
+ with qos_file_path.open() as f:
+ return float(f.read().strip())
diff --git a/hpvm/projects/hpvm-profiler/setup.py b/hpvm/projects/hpvm-profiler/setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7b3771a2ee3a67569092d9b956e67c309f9d08e
--- /dev/null
+++ b/hpvm/projects/hpvm-profiler/setup.py
@@ -0,0 +1,11 @@
+import setuptools
+
+setuptools.setup(
+ name="hpvm_profiler",
+ version="0.1",
+ author="Akash Kothari, Yifan Zhao",
+ author_email="akashk4@illinois.edu, yifanz16@illinois.edu",
+ description="A package for profiling of HPVM approximation configurations",
+ packages=["hpvm_profiler"],
+ install_requires=["numpy>=1.19", "matplotlib>=3"],
+)
diff --git a/hpvm/scripts/hpvm_installer.py b/hpvm/scripts/hpvm_installer.py
index cd6ef3bae08b0cd6c912a40b6a45e3ca37257c99..e83d5f3e727044dcba942e2533597bc779c9b816 100755
--- a/hpvm/scripts/hpvm_installer.py
+++ b/hpvm/scripts/hpvm_installer.py
@@ -35,6 +35,7 @@ MAKE_TEST_TARGETS = ["check-hpvm-dnn", "check-hpvm-pass"]
# Relative to project root which is __file__.parent.parent
PY_PACKAGES = [
+ "projects/hpvm-profiler",
"projects/predtuner",
"projects/torch2hpvm",
"projects/keras",
diff --git a/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnn.py b/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnn.py
deleted file mode 100644
index 10eaf5699f5549d449fab12bd8a5ed9e8a5d2d6e..0000000000000000000000000000000000000000
--- a/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnn.py
+++ /dev/null
@@ -1,210 +0,0 @@
-import os.path
-from os import path
-import sys
-import matplotlib.pyplot as plt
-
-
-binary_dir = "../../../build/tools/hpvm/test/dnn_benchmarks/"
-tradeoff_curves_dir = "../../../docs/tradeoff-curves"
-
-
-accuracy_file = "final_accuracy"
-profile_file = "profile_data.txt"
-profile_file_prefix = "profile_info_"
-
-temp_file_name = "temp.txt"
-pred_binary_prefix = ""
-pred_binary_suffix = ""
-
-rt_binary_suffix = "_rt_pred"
-max_num_runs = 20
-
-
-def max_num_configs (config_file):
- num_configs = 0
- with open(config_file, "r") as f:
- for line in f:
- if "conf" in line:
- num_configs = num_configs + 1
- return (num_configs + 1)
-
-
-def read_and_write_config (config_file, config_num, temp_file):
- config = ""
- print("--CONFIG FILE: " + config_file)
- print("--CONFIG NUM: " + str(config_num))
- print("--TEMP FILE: " + temp_file)
- with open(config_file, "r") as f:
- conf = "conf" + str(config_num)
- read_config = False
- read_first_line = False
- for line in f:
- if read_first_line == False:
- config = config + line
- read_first_line = True
- continue
- if "-----" in line and read_config == True:
- read_config = False
- config = config + line
- break
- if read_config == True:
- config = config + line
- continue
- if conf in line:
- read_config = True
- config = config + "+++++\n"
- config = config + line
- print("config: ")
- print(config)
- with open(temp_file, "w") as f:
- f.write(config)
-
-
-def get_avg_exec_time(profile_file_path, config_num):
- prof_file = profile_file_path + profile_file_prefix + str(config_num) + ".txt"
- print("PROFILE FILE: " + prof_file)
- with open(prof_file, "r") as f:
- for line in f:
- if "Total Time" in line:
- print("LINE: " + line)
- time = line.strip().split() [3]
- print("TIME: " + time)
- return float(time)
- print("ERROR")
- sys.exit()
- return float(-1)
-
-def get_exec_time(config_file):
- print("CONFIG FILE: " + config_file)
- with open(config_file, "r") as f:
- for line in f:
- if "conf" in line:
- print("LINE: " + line)
- time = line.strip().split() [1]
- print("TIME: " + time)
- return float(time)
- print("ERROR")
- sys.exit()
- return float(-1)
-
-def get_avg_exec_accuracy(file_name):
- with open(file_name, "r") as f:
- for line in f:
- accuracy = line.strip().split() [0]
- print("ACCURACY: " + accuracy)
- return float(accuracy)
- print("ERROR")
- sys.exit()
- return float(-1)
-
-def get_exec_accuracy(config_file):
- with open(config_file, "r") as f:
- for line in f:
- if "conf" in line:
- print("LINE: " + line)
- acc = line.strip().split() [4]
- print("ACCURACY: " + acc)
- return float(acc)
- print("ERROR")
- sys.exit()
- return float(-1)
-
-def predictive_tuning_exec(dnn_name):
- dnn_dir = "../benchmarks/" + dnn_name
- binary_name = binary_dir + pred_binary_prefix + dnn_name + pred_binary_suffix
- pred_dir = dnn_dir + "/predictive/"
- config_file = pred_dir + dnn_name + ".txt"
- temp_file = pred_dir + temp_file_name
- print("dnn_dir: " + dnn_dir)
- print("binary name: " + binary_name)
- print("pred_dir: " + pred_dir)
- print("config_file: " + config_file)
- print("temp_file: " + temp_file)
- exec_command = "rm " + temp_file + " " + accuracy_file + " " + profile_file + " " + pred_dir + "profile*"
- print(exec_command)
- os.system(exec_command)
- config_num = 1
- max_configs = max_num_configs(config_file)
- baseline_time = 0
- baseline_acc = 0
- print("MAX CONFIGS: " + str(max_configs))
- perf_list = list()
- acc_list = list()
- while config_num < max_configs:
- read_and_write_config(config_file, config_num, temp_file)
- exec_command = binary_name
- print(exec_command)
- os.system(exec_command)
- time = get_avg_exec_time(pred_dir, config_num - 1)
- acc = get_avg_exec_accuracy(accuracy_file)
- config_time = get_exec_time(temp_file)
- config_acc = get_exec_accuracy(temp_file)
- if config_num == 1:
- baseline_time = time
- baseline_acc = acc
- else:
- print("SPEEDUP: ")
- print(baseline_time/time)
- perf_list.append(baseline_time/time)
- print("CONFIG TIME: ")
- print(config_time)
- print("ACC LOSS: ")
- print(baseline_acc - acc)
- acc_list.append(baseline_acc - acc)
- print("CONFIG ACC: ")
- print(config_acc)
- config_num = config_num + 1
- exec_command = "rm " + temp_file + " " + accuracy_file + " " + profile_file + " " + pred_dir + "profile*"
- print(exec_command)
- os.system(exec_command)
- plt.scatter(acc_list, perf_list)
- plt.ylabel("Speedup (X)")
- plt.xlabel("Accurancy loss (%)")
- xticks = ['-1', '0', '1', '2', '3', '4']
- yticks = ['1', '1.5', '2', '2.5', '3']
- plt.xlim(-1, 4)
- plt.ylim(1, 3)
- plt.title(dnn_name)
- plt.savefig(tradeoff_curves_dir + dnn_name + "_tradeoff.pdf")
- perf_list.clear()
- acc_list.clear()
-
-
-def runtime_tuning_exec():
- num_args = len(sys.argv)
- binary_files = list()
- arg = 2
- while arg < num_args:
- binary_files.append(sys.argv[arg])
- arg = arg + 1
-
- for dnn_name in binary_files:
- binary_dir = "../benchmarks/" + dnn_name
- binary_name = binary_dir + rt_binary_suffix
- conf_dir = binary_dir + "/data"
- print("binary_dir: " + binary_dir)
- print("binary name: " + binary_name)
- run = 0
- while run < max_num_runs:
- exec_command = binary_name
- print(exec_command)
- os.system(exec_command)
- exec_command = "/home/nvidia/poll 13"
- print(exec_command)
- os.system(exec_command)
- exec_command = "mv " + conf_dir + "/profile_info_0.txt " + conf_dir + "/profile_info_out-run-" + str(run) + ".txt"
- print(exec_command)
- os.system(exec_command)
- run = run + 1
- exec_command = "rm -rf " + conf_dir + "/run_data"
- print(exec_command)
- os.system(exec_command)
- exec_command = "mkdir " + conf_dir + "/run_data"
- print(exec_command)
- os.system(exec_command)
-
-
-
-if __name__ == "__main__":
- predictive_tuning_exec(sys.argv[1])
-
diff --git a/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnns.py b/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnns.py
deleted file mode 100644
index 4c9ab6ec12741d05d5af6b414a90ad26c5d1bfe0..0000000000000000000000000000000000000000
--- a/hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnns.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import os
-import sys
-
-dnns = ["alexnet", "alexnet2", "vgg16_cifar10", "vgg16_cifar100", "resnet18", "mobilenet_cifar10", "alexnet_imagenet", "resnet50_imagenet", "vgg16_imagenet", "lenet_mnist"]
-
-
-if __name__ == "__main__":
- for dnn in dnns:
- exec_command = "python3 run_dnn.py " + dnn
- print(exec_command)
- os.system(exec_command)
-
diff --git a/hpvm/test/dnn_benchmarks/hpvm-c/scripts/README.md b/hpvm/test/dnn_benchmarks/profiling/README.md
similarity index 100%
rename from hpvm/test/dnn_benchmarks/hpvm-c/scripts/README.md
rename to hpvm/test/dnn_benchmarks/profiling/README.md
diff --git a/hpvm/test/dnn_benchmarks/hpvm-c/scripts/jetson_clocks.sh b/hpvm/test/dnn_benchmarks/profiling/jetson_clocks.sh
similarity index 100%
rename from hpvm/test/dnn_benchmarks/hpvm-c/scripts/jetson_clocks.sh
rename to hpvm/test/dnn_benchmarks/profiling/jetson_clocks.sh