Converted the hpvm profiler into a package

hpvm/projects/hpvm-profiler/hpvm_profiler/__init__.py

+from pathlib import Path
+from typing import Iterable, List, Tuple, Union
+from dataclasses import dataclass
+
+PathLike = Union[Path, str]
+
+
+def profile_configs(
+    binary_path: PathLike,
+    config_path: PathLike,
+    output_config_path: PathLike,
+    profile_filename: str = "profile_info.txt",
+    qos_filename: str = "final_accuracy",
+):
+    """
+    Profile an HPVM configuration file with an HPVM binary.
+    The configuration file must have the baseline as the first configuration.
+
+    binary_path: Union[Path, str]
+        Path to binary to be executed in profiling.
+    config_path: Union[Path, str]
+        Path to config file (HPVM configuration format)
+        with configs to enumerate for profiling.
+    output_config_path: Union[Path, str]
+        Path where the output configs are written.
+        The output config file has the same configs as the input `config_path` file,
+        but the performance and energy readings are updated.
+    profile_filename: str
+        Name of profile file generated by the binary (in current directory).
+        This defaults to "profile_info.txt" and should not be changed for HPVM binaries.
+    qos_filename: str
+        Name of QoS file generated by the binary (in current directory).
+        It contains a single float number as the QoS of this run.
+        This defaults to "final_accuracy" and should not be changed for HPVM binaries.
+    """
+
+    from subprocess import check_call
+    from tempfile import NamedTemporaryFile
+
+    # Read first line ("the float") and configs in config file
+    header, configs = read_hpvm_configs(Path(config_path))
+    if not configs:
+        raise ValueError("Config file with no configs is unsupported.")
+    temp_file = NamedTemporaryFile("w")
+    baseline_time, baseline_acc = None, None
+    for idx, config in enumerate(configs):
+        # Write config to temp config file
+        write_hpvm_config(header, [config], Path(temp_file.name))
+        # Run binary_path binary,
+        # which generates `profile_filename` and `qos_filename` file in cwd.
+        check_call(str(binary_path))
+        # Read these two files for time and QoS info.
+        time = _read_profile_file(Path(profile_filename))
+        acc = _read_qos_file(Path(qos_filename))
+        if idx == 0:
+            baseline_time, baseline_acc = time, acc
+            continue
+        assert baseline_time is not None and baseline_acc is not None
+        speedup = baseline_time / time
+        config.update_profile_results(speedup, acc, baseline_acc)
+    write_hpvm_config(header, configs, Path(output_config_path))
+    temp_file.close()
+
+
+def plot_hpvm_configs(
+    config_path: PathLike,
+    save_to: PathLike = None,
+    show_qos_loss: bool = True,
+    **fig_kwargs,
+):
+    """
+    Plot the QoS-speedup information in an HPVM configuration file.
+    It is recommended to profile the config file first (using `profile_configs`)
+    to obtain real speedup numbers.
+    This function creates a `matplotlib.pyplot.Figure`, plots on it, and returns it.
+
+    config_path: Union[Path, str]
+        Path to the config file (HPVM configuration format).
+    save_to: Union[Path, str]
+        File to save figure into. Default is None: don't save figure (just return it).
+    show_qos_loss: bool
+        Show the loss of QoS on x axis of the figure. Defaults to True.
+        If False, will use (absolute) QoS instead of QoS loss.
+    fig_kwargs:
+        Arguments to pass to `plt.subplots`.
+    """
+
+    import numpy as np
+    import matplotlib.pyplot as plt
+
+    _, configs = read_hpvm_configs(config_path)
+    get_qos = lambda c: c.qos_loss if show_qos_loss else c.qos
+    qos_speedup = np.array([(get_qos(c), c.speedup) for c in configs])
+    qoses, speedups = qos_speedup.T
+    fig, ax = plt.subplots(**fig_kwargs)
+    ax.scatter(qoses, speedups)
+    ax.xlabel("QoS Loss")
+    ax.ylabel("Speedup (X)")
+    if save_to:
+        fig.savefig(save_to, dpi=300)
+    return fig
+
+@dataclass
+class Config:
+    conf_name: str
+    speedup: float
+    energy: float
+    qos: float
+    qos_loss: float
+    # We don't care about the information in this part, and we don't parse this.
+    config_body: List[str]
+
+    def update_profile_results(self, speedup: float, qos: float, base_qos: float):
+        recorded_base_qos = self.qos + self.qos_loss
+        if abs(recorded_base_qos - base_qos) > 1e-3:
+            raise ValueError(
+                f"Baseline QoS mismatch. Original: {recorded_base_qos}, measured: {base_qos}"
+            )
+        self.speedup = speedup
+        self.qos = qos
+        self.qos_loss = base_qos - qos
+
+    def __repr__(self) -> str:
+        header_fields = [
+            self.conf_name,
+            self.speedup,
+            self.energy,
+            self.qos,
+            self.qos_loss,
+        ]
+        header = " ".join(str(field) for field in header_fields)
+        return f"{header}\n{self.config_body}"
+
+    __str__ = __repr__
+
+
+def read_hpvm_configs(config_file: PathLike) -> Tuple[str, List[Config]]:
+    # def read_hpvm_configs(config_file, config_num, temp_file):
+    ret_configs = []
+    with open(config_file) as f:
+        text = f.read()
+    opening, closing = "+++++", "-----"
+    # There's 1 float sitting on the first line of config file.
+    # We don't use it, but want to keep that intact.
+    header, *configs = text.split(opening)
+    header = header.strip()
+    for config_text in configs:
+        config_text = config_text.replace(closing, "").strip()
+        config_header, *config_body = config_text.splitlines()
+        conf_name, *number_fields = config_header.split(" ")
+        speedup, energy, qos, qos_drop = [float(s) for s in number_fields]
+        ret_configs.append(
+            Config(conf_name, speedup, energy, qos, qos_drop, config_body)
+        )
+    return header, ret_configs
+
+
+def write_hpvm_config(header: str, configs: Iterable[Config], to_file: PathLike):
+    text_segs = [header] + [str(config) for config in configs]
+    with open(to_file, "w") as f:
+        f.write("\n".join(text_segs))
+
+
+def _read_profile_file(profile_file_path: Path):
+    with profile_file_path.open() as f:
+        target_lines = [line.strip() for line in f if "Total Time" in line]
+    if len(target_lines) != 1:
+        raise RuntimeError(f"Profile {profile_file_path} malformed")
+    (target_line,) = target_lines
+    return float(target_line.split()[3])
+
+
+def _read_qos_file(qos_file_path: Path):
+    with qos_file_path.open() as f:
+        return float(f.read().strip())

hpvm/projects/hpvm-profiler/setup.py

+import setuptools
+
+setuptools.setup(
+    name="hpvm_profiler",
+    version="0.1",
+    author="Akash Kothari, Yifan Zhao",
+    author_email="akashk4@illinois.edu, yifanz16@illinois.edu",
+    description="A package for profiling of HPVM approximation configurations",
+    packages=["hpvm_profiler"],
+    install_requires=["numpy>=1.19", "matplotlib>=3"],
+)

hpvm/scripts/hpvm_installer.py

@@ -35,6 +35,7 @@ MAKE_TEST_TARGETS = ["check-hpvm-dnn", "check-hpvm-pass"]
 
 # Relative to project root which is __file__.parent.parent
 PY_PACKAGES = [
+    "projects/hpvm-profiler",
     "projects/predtuner",
     "projects/torch2hpvm",
     "projects/keras",

hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnn.py

-import os.path
-from os import path
-import sys
-import matplotlib.pyplot as plt 
-
-
-binary_dir = "../../../build/tools/hpvm/test/dnn_benchmarks/"
-tradeoff_curves_dir = "../../../docs/tradeoff-curves"
-
-
-accuracy_file = "final_accuracy"
-profile_file = "profile_data.txt"
-profile_file_prefix = "profile_info_"
-
-temp_file_name = "temp.txt"
-pred_binary_prefix = ""
-pred_binary_suffix = "" 
-
-rt_binary_suffix = "_rt_pred"
-max_num_runs = 20
-
-
-def max_num_configs (config_file):
-    num_configs = 0
-    with open(config_file, "r") as f:
-        for line in f:
-            if "conf" in line:
-                num_configs = num_configs + 1
-    return (num_configs + 1)
-
-
-def read_and_write_config (config_file, config_num, temp_file):
-    config = ""
-    print("--CONFIG FILE: " + config_file)
-    print("--CONFIG NUM: " + str(config_num))
-    print("--TEMP FILE: " + temp_file)
-    with open(config_file, "r") as f:
-        conf = "conf" + str(config_num)
-        read_config = False
-        read_first_line = False
-        for line in f:
-            if read_first_line == False:
-                config = config + line
-                read_first_line = True
-                continue
-            if "-----" in line and read_config == True:
-                read_config = False
-                config = config + line
-                break
-            if read_config == True:
-                config = config + line
-                continue
-            if conf in line:
-                read_config = True
-                config = config + "+++++\n"
-                config = config + line
-    print("config: ")
-    print(config)
-    with open(temp_file, "w") as f:
-        f.write(config)
-
-
-def get_avg_exec_time(profile_file_path, config_num):
-    prof_file = profile_file_path + profile_file_prefix + str(config_num) + ".txt"
-    print("PROFILE FILE: " + prof_file)
-    with open(prof_file, "r") as f:
-            for line in f:
-                if "Total Time" in line:
-                    print("LINE: " + line)
-                    time = line.strip().split() [3]
-                    print("TIME: " + time)
-                    return float(time)
-    print("ERROR")
-    sys.exit()
-    return float(-1)
-
-def get_exec_time(config_file):
-    print("CONFIG FILE: " + config_file)
-    with open(config_file, "r") as f:
-        for line in f:
-            if "conf" in line:
-                print("LINE: " + line)
-                time = line.strip().split() [1]
-                print("TIME: " + time)
-                return float(time)
-    print("ERROR")
-    sys.exit()
-    return float(-1)
-
-def get_avg_exec_accuracy(file_name):
-    with open(file_name, "r") as f:
-        for line in f:
-            accuracy = line.strip().split() [0]
-            print("ACCURACY: " + accuracy)
-            return float(accuracy)
-    print("ERROR")
-    sys.exit()
-    return float(-1)
-
-def get_exec_accuracy(config_file):
-    with open(config_file, "r") as f:
-        for line in f:
-            if "conf" in line:
-                print("LINE: " + line)
-                acc = line.strip().split() [4]
-                print("ACCURACY: " + acc)
-                return float(acc)
-    print("ERROR")
-    sys.exit()
-    return float(-1)
-
-def predictive_tuning_exec(dnn_name):
-    dnn_dir = "../benchmarks/" + dnn_name
-    binary_name = binary_dir + pred_binary_prefix + dnn_name + pred_binary_suffix
-    pred_dir = dnn_dir + "/predictive/"
-    config_file = pred_dir + dnn_name + ".txt"
-    temp_file = pred_dir + temp_file_name
-    print("dnn_dir: " + dnn_dir)
-    print("binary name: " + binary_name)
-    print("pred_dir: " + pred_dir)
-    print("config_file: " + config_file)
-    print("temp_file: " + temp_file)
-    exec_command = "rm " + temp_file + " " + accuracy_file + " " + profile_file + " " + pred_dir + "profile*"
-    print(exec_command)
-    os.system(exec_command)
-    config_num = 1
-    max_configs = max_num_configs(config_file)
-    baseline_time = 0
-    baseline_acc = 0
-    print("MAX CONFIGS: " + str(max_configs))
-    perf_list = list()
-    acc_list = list()
-    while config_num < max_configs:
-        read_and_write_config(config_file, config_num, temp_file)
-        exec_command = binary_name
-        print(exec_command) 
-        os.system(exec_command)
-        time = get_avg_exec_time(pred_dir, config_num - 1)
-        acc = get_avg_exec_accuracy(accuracy_file)
-        config_time = get_exec_time(temp_file)
-        config_acc = get_exec_accuracy(temp_file)
-        if config_num == 1:
-            baseline_time = time
-            baseline_acc = acc 
-        else:
-            print("SPEEDUP: ")
-            print(baseline_time/time)
-            perf_list.append(baseline_time/time)
-            print("CONFIG TIME: ")
-            print(config_time)
-            print("ACC LOSS: ")
-            print(baseline_acc - acc)
-            acc_list.append(baseline_acc - acc)
-            print("CONFIG ACC: ")
-            print(config_acc)
-        config_num = config_num + 1
-    exec_command = "rm " + temp_file + " " + accuracy_file + " " + profile_file + " " + pred_dir + "profile*"
-    print(exec_command)
-    os.system(exec_command)
-    plt.scatter(acc_list, perf_list)
-    plt.ylabel("Speedup (X)")
-    plt.xlabel("Accurancy loss (%)")
-    xticks = ['-1', '0', '1', '2', '3', '4']
-    yticks = ['1', '1.5', '2', '2.5', '3']
-    plt.xlim(-1, 4)
-    plt.ylim(1, 3)
-    plt.title(dnn_name)
-    plt.savefig(tradeoff_curves_dir + dnn_name + "_tradeoff.pdf")
-    perf_list.clear()
-    acc_list.clear()
-
-
-def runtime_tuning_exec():
-    num_args = len(sys.argv)
-    binary_files = list()
-    arg = 2
-    while arg < num_args:
-        binary_files.append(sys.argv[arg])
-        arg = arg + 1
-
-    for dnn_name in binary_files:
-        binary_dir = "../benchmarks/" + dnn_name
-        binary_name = binary_dir + rt_binary_suffix
-        conf_dir = binary_dir + "/data"
-        print("binary_dir: " + binary_dir)
-        print("binary name: " + binary_name)
-        run = 0
-        while run < max_num_runs:
-            exec_command = binary_name
-            print(exec_command)
-            os.system(exec_command)
-            exec_command = "/home/nvidia/poll 13"
-            print(exec_command)
-            os.system(exec_command)
-            exec_command = "mv " + conf_dir + "/profile_info_0.txt " + conf_dir + "/profile_info_out-run-" + str(run) + ".txt"
-            print(exec_command)
-            os.system(exec_command)
-            run = run + 1
-        exec_command = "rm -rf " + conf_dir +  "/run_data"
-        print(exec_command)
-        os.system(exec_command)
-        exec_command = "mkdir " + conf_dir + "/run_data"  
-        print(exec_command)
-        os.system(exec_command)  
-            
-
-
-if __name__ == "__main__":
-    predictive_tuning_exec(sys.argv[1])
-

hpvm/test/dnn_benchmarks/hpvm-c/scripts/run_dnns.py

-import os
-import sys
-
-dnns = ["alexnet", "alexnet2", "vgg16_cifar10", "vgg16_cifar100", "resnet18", "mobilenet_cifar10", "alexnet_imagenet", "resnet50_imagenet", "vgg16_imagenet", "lenet_mnist"]
-
-
-if __name__ == "__main__":
-    for dnn in dnns:
-        exec_command = "python3 run_dnn.py " + dnn 
-        print(exec_command)
-        os.system(exec_command)
-