Added test cases for tuning (coming from torch frontend)

hpvm/test/dnn_benchmarks/pytorch/CMakeLists.txt

-# --[ llvm-lit test setup
-# lit.cfg.py looks for tests in CMAKE_CURRENT_BINARY_DIR (see lit.cfg.py)
-# as most of the tests require some kind of compilation / generation
-# which is best done over there.
+# --[ llvm-lit test setup for test_frontend/
 configure_lit_site_cfg(
   ../../lit.site.cfg.py.in
-  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+  ${CMAKE_CURRENT_BINARY_DIR}/test_frontend/lit.site.cfg.py
   MAIN_CONFIG
-  ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+  ${CMAKE_CURRENT_SOURCE_DIR}/test_frontend/lit.cfg.py
 )
-add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for package torch2hpvm"
-  ${CMAKE_CURRENT_BINARY_DIR}
+add_lit_testsuite(check-hpvm-torch2hpvm "Run tests for HPVM PyTorch frontend"
+  ${CMAKE_CURRENT_BINARY_DIR}/test_frontend
   # We depend on check_dnn_acc.py defined in ../hpvm-c/
   # to compare the inference accuracy of our frontend-generated binary
   # to that of the baseline.
   DEPENDS check_dnn_acc
   ARGS "-j1"  # Run frontend generation sequentially
 )
+
+# --[ llvm-lit test setup for test_tuning/
+configure_lit_site_cfg(
+  ../../lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/test_tuning/lit.site.cfg.py
+  MAIN_CONFIG
+  ${CMAKE_CURRENT_SOURCE_DIR}/test_tuning/lit.cfg.py
+)
+add_lit_testsuite(check-hpvm-tuning "Run tests for autotuning procedure"
+  ${CMAKE_CURRENT_BINARY_DIR}/test_tuning
+  ARGS "-j1"  # Run tuning tests sequentially
+)

hpvm/test/dnn_benchmarks/pytorch/test_frontend.py → hpvm/test/dnn_benchmarks/pytorch/test_frontend/test_frontend.py

 #!/usr/bin/env python3
-import os
 import shutil
 import site
 from pathlib import Path
@@ -10,7 +9,7 @@ import torch
 from torch2hpvm import BinDataset, ModelExporter
 from torch.nn import Module
 
-site.addsitedir(os.path.dirname(__file__))
+site.addsitedir(Path(__file__).parent.parent.absolute())
 import dnn
 
 benchmarks = {
@@ -33,7 +32,7 @@ print(f"Generating {netname} to {codegen_dir}")
 if codegen_dir.exists():
     shutil.rmtree(codegen_dir)
 
-params = self_folder / "../model_params" / netname
+params = self_folder / "../../model_params" / netname
 dataset_shape = 5000, nch, img_size, img_size
 bin_tuneset = BinDataset(
     params / "tune_input.bin", params / "tune_labels.bin", dataset_shape

hpvm/test/dnn_benchmarks/pytorch/test_tuning/alexnet_imagenet.test

+RUN: test_tuning.py alexnet_imagenet 0

hpvm/test/dnn_benchmarks/pytorch/test_tuning/lenet_mnist_p1.test

+RUN: test_tuning.py lenet_mnist 1

hpvm/test/dnn_benchmarks/pytorch/test_tuning/lit.cfg.py

+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+
+import lit.formats
+from lit.llvm import llvm_config
+
+# name: The name of this test suite.
+config.name = "HPVM-Predtuner"
+
+# testFormat: The test format to use to interpret tests.
+config.test_format = lit.formats.ShTest(False)
+
+# suffixes: A list of file extensions to treat as test files. This is overriden
+# by individual lit.local.cfg files in the test subdirectories.
+config.suffixes = [".test"]
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# test_exec_root: The root path where tests should be run.
+current_source_dir = os.path.dirname(os.path.relpath(__file__, config.llvm_src_root))
+current_binary_dir = os.path.join(config.llvm_obj_root, current_source_dir)
+config.test_exec_root = current_binary_dir
+
+# Tweak the PATH to include the tools dir.
+llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
+
+# Add substitution for our main script in this directory.
+llvm_config.add_tool_substitutions(["test_tuning.py"], config.test_source_root)

hpvm/test/dnn_benchmarks/pytorch/test_tuning/mobilenet_cifar10.test

+RUN: test_tuning.py mobilenet_cifar10 0

hpvm/test/dnn_benchmarks/pytorch/test_tuning/resnet18_cifar10.test

+RUN: test_tuning.py resnet18_cifar10 0

hpvm/test/dnn_benchmarks/pytorch/test_tuning.py → hpvm/test/dnn_benchmarks/pytorch/test_tuning/test_tuning.py

-import os
+#!/usr/bin/env python3
 import shutil
 import site
 from pathlib import Path
+from sys import argv
 
 import torch
 from predtuner import config_pylogger
@@ -9,41 +10,31 @@ from predtuner.pipedbin import PipedBinaryApp
 from torch2hpvm import BinDataset, ModelExporter
 from torch.nn import Module
 
-site.addsitedir(os.path.dirname(__file__))
+site.addsitedir(Path(__file__).parent.parent.absolute())
 import dnn
 
 # Set up logger to put log file in /tmp
 msg_logger = config_pylogger(output_dir="/tmp", verbose=True)
+benchmarks = {
+    "lenet_mnist": (dnn.LeNet, 1, 28, 1000),
+    "alexnet_imagenet": (dnn.AlexNetImageNet, 3, 224, 100),
+    "mobilenet_cifar10": (dnn.MobileNet, 3, 32, 500),
+    "resnet18_cifar10": (dnn.ResNet18, 3, 32, 500),
+    "vgg16_cifar10": (dnn.VGG16Cifar10, 3, 32, 500),
+}
+model_param = Path(__file__).parent / "../../model_params"
 
 
-benchmarks = [
-    (dnn.LeNet, 1, 28, 500, "lenet_mnist"),
-    (dnn.AlexNet, 3, 32, 500, "alexnet_cifar10"),
-    (dnn.AlexNet2, 3, 32, 500, "alexnet2_cifar10"),
-    (dnn.AlexNetImageNet, 3, 224, 100, "alexnet_imagenet"),
-    (dnn.MobileNet, 3, 32, 500, "mobilenet_cifar10"),
-    (dnn.ResNet18, 3, 32, 500, "resnet18_cifar10"),
-    (dnn.ResNet50, 3, 224, 50, "resnet50_imagenet"),
-    (dnn.VGG16Cifar10, 3, 32, 500, "vgg16_cifar10"),
-    (dnn.VGG16Cifar100, 3, 32, 500, "vgg16_cifar100"),
-    (dnn.VGG16ImageNet, 3, 224, 50, "vgg16_imagenet"),
-]
-model_param = Path(__file__).parent / "../model_params"
-
-
-def generate(model_cls, nch, img_size, batch_size, pathname):
-    codegen_dir = Path(f"/tmp/{pathname}_tune")
+def generate(model_cls, nch, img_size, batch_size, netname):
+    codegen_dir = Path(f"./{netname}")
     build_dir = codegen_dir / "build"
     metadata_file = codegen_dir / "ops.json"
-    binary_file = build_dir / pathname
+    binary_file = build_dir / netname
     build_dir = codegen_dir / "build"
-    # if binary_file.is_file() and metadata_file.is_file():
-    #     return binary_file, metadata_file
 
-    print(f"Generating {pathname} to {codegen_dir}")
     if codegen_dir.exists():
         shutil.rmtree(codegen_dir)
-    params = model_param / pathname
+    params = model_param / netname
     dataset_shape = 5000, nch, img_size, img_size
     bin_tuneset = BinDataset(
         params / "tune_input.bin", params / "tune_labels.bin", dataset_shape
@@ -52,7 +43,7 @@ def generate(model_cls, nch, img_size, batch_size, pathname):
         params / "test_input.bin", params / "test_labels.bin", dataset_shape
     )
     model: Module = model_cls()
-    checkpoint = model_param / f"{pathname}.pth.tar"
+    checkpoint = model_param / f"pytorch/{netname}.pth.tar"
     model.load_state_dict(torch.load(checkpoint.as_posix()))
     exporter = ModelExporter(
         model, bin_tuneset, bin_testset, codegen_dir, target="hpvm_tensor_inspect"
@@ -62,19 +53,27 @@ def generate(model_cls, nch, img_size, batch_size, pathname):
 
 
 def main():
-    for model_cls, nch, img_size, batch_size, pathname in benchmarks:
-        print(f"Testing {pathname}")
-        binary_file, metadata_file = generate(
-            model_cls, nch, img_size, batch_size, pathname
-        )
-        app = PipedBinaryApp("test", binary_file, metadata_file)
-        tuner = app.get_tuner()
-        tuner.tune(100, 3.0, 3.0, True, 50, cost_model="cost_linear")
-        tuner.dump_configs("configs.json")
-        fig = tuner.plot_configs(show_qos_loss=True)
-        fig.savefig("configs.png", dpi=300)
-        app.dump_hpvm_configs(tuner.best_configs, "hpvm_confs.txt")
+    netname, is_pred = argv[1:]
+    is_pred = int(is_pred)
+    model_cls, nch, img_size, batch_size = benchmarks[netname]
+    binary_file, metadata_file = generate(
+        model_cls, nch, img_size, batch_size, netname
+    )
+    print(Path.cwd(), binary_file, metadata_file)
+    app = PipedBinaryApp("test", binary_file, metadata_file)
+    tuner = app.get_tuner()
+    tuner.tune(
+        5,
+        3.0,
+        is_threshold_relative=True,
+        cost_model="cost_linear",
+        qos_model="qos_p1" if is_pred else "none",
+    )
+    tuner.dump_configs("configs.json")
+    fig = tuner.plot_configs(show_qos_loss=True)
+    fig.savefig("configs.png", dpi=300)
+    app.dump_hpvm_configs(tuner.best_configs, "hpvm_confs.txt")
 
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()

hpvm/test/dnn_benchmarks/pytorch/test_tuning/vgg16_cifar10.test

+RUN: test_tuning.py vgg16_cifar10 0